2024-11-14 09:13:34,774 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-14 09:13:34,785 main DEBUG Took 0.009292 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-14 09:13:34,785 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-14 09:13:34,786 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-14 09:13:34,786 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-14 09:13:34,788 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:13:34,794 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-14 09:13:34,805 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:13:34,807 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:13:34,807 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:13:34,807 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:13:34,808 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:13:34,808 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:13:34,809 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:13:34,809 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:13:34,810 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:13:34,810 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:13:34,810 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:13:34,811 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:13:34,811 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:13:34,811 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:13:34,812 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:13:34,812 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:13:34,813 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:13:34,813 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:13:34,813 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:13:34,813 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:13:34,814 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:13:34,814 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:13:34,814 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:13:34,815 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:13:34,815 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:13:34,815 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-14 09:13:34,817 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:13:34,818 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-14 09:13:34,819 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-14 09:13:34,820 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-14 09:13:34,821 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-14 09:13:34,821 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-14 09:13:34,829 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-14 09:13:34,831 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-14 09:13:34,833 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-14 09:13:34,833 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-14 09:13:34,834 main DEBUG createAppenders(={Console}) 2024-11-14 09:13:34,834 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-14 09:13:34,835 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-14 09:13:34,835 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-14 09:13:34,835 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-14 09:13:34,836 main DEBUG OutputStream closed 2024-11-14 09:13:34,836 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-14 09:13:34,836 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-14 09:13:34,836 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-14 09:13:34,901 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-14 09:13:34,903 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-14 09:13:34,904 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-14 09:13:34,905 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-14 09:13:34,906 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-14 09:13:34,906 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-14 09:13:34,906 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-14 09:13:34,906 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-14 09:13:34,907 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-14 09:13:34,907 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-14 09:13:34,907 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-14 09:13:34,907 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-14 09:13:34,908 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-14 09:13:34,908 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-14 09:13:34,908 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-14 09:13:34,908 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-14 09:13:34,909 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-14 09:13:34,909 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-14 09:13:34,912 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-14 09:13:34,912 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-14 09:13:34,912 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-14 09:13:34,913 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-14T09:13:35,126 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae915b1d-3a0c-f123-b058-6fcd2c888fa8 2024-11-14 09:13:35,129 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-14 09:13:35,129 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-14T09:13:35,137 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-14T09:13:35,155 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-14T09:13:35,170 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae915b1d-3a0c-f123-b058-6fcd2c888fa8/cluster_e7e5a44a-d236-23e3-0b04-c3f634c73d33, deleteOnExit=true 2024-11-14T09:13:35,170 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-14T09:13:35,171 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae915b1d-3a0c-f123-b058-6fcd2c888fa8/test.cache.data in system properties and HBase conf 2024-11-14T09:13:35,172 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae915b1d-3a0c-f123-b058-6fcd2c888fa8/hadoop.tmp.dir in system properties and HBase conf 2024-11-14T09:13:35,172 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae915b1d-3a0c-f123-b058-6fcd2c888fa8/hadoop.log.dir in system properties and HBase conf 2024-11-14T09:13:35,173 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae915b1d-3a0c-f123-b058-6fcd2c888fa8/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-14T09:13:35,173 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae915b1d-3a0c-f123-b058-6fcd2c888fa8/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-14T09:13:35,174 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-14T09:13:35,263 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-14T09:13:35,354 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T09:13:35,358 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae915b1d-3a0c-f123-b058-6fcd2c888fa8/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-14T09:13:35,358 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae915b1d-3a0c-f123-b058-6fcd2c888fa8/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-14T09:13:35,359 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae915b1d-3a0c-f123-b058-6fcd2c888fa8/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-14T09:13:35,359 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae915b1d-3a0c-f123-b058-6fcd2c888fa8/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-14T09:13:35,360 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae915b1d-3a0c-f123-b058-6fcd2c888fa8/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-14T09:13:35,360 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae915b1d-3a0c-f123-b058-6fcd2c888fa8/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-14T09:13:35,360 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae915b1d-3a0c-f123-b058-6fcd2c888fa8/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-14T09:13:35,361 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae915b1d-3a0c-f123-b058-6fcd2c888fa8/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-14T09:13:35,361 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae915b1d-3a0c-f123-b058-6fcd2c888fa8/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-14T09:13:35,361 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae915b1d-3a0c-f123-b058-6fcd2c888fa8/nfs.dump.dir in system properties and HBase conf 2024-11-14T09:13:35,362 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae915b1d-3a0c-f123-b058-6fcd2c888fa8/java.io.tmpdir in system properties and HBase conf 2024-11-14T09:13:35,362 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae915b1d-3a0c-f123-b058-6fcd2c888fa8/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-14T09:13:35,363 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae915b1d-3a0c-f123-b058-6fcd2c888fa8/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-14T09:13:35,363 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae915b1d-3a0c-f123-b058-6fcd2c888fa8/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-14T09:13:37,597 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-14T09:13:37,700 INFO [Time-limited test {}] log.Log(170): Logging initialized @3637ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-14T09:13:37,793 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T09:13:37,852 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T09:13:37,872 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T09:13:37,872 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T09:13:37,873 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-14T09:13:37,888 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T09:13:37,891 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae915b1d-3a0c-f123-b058-6fcd2c888fa8/hadoop.log.dir/,AVAILABLE} 2024-11-14T09:13:37,891 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T09:13:38,081 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae915b1d-3a0c-f123-b058-6fcd2c888fa8/java.io.tmpdir/jetty-localhost-35919-hadoop-hdfs-3_4_1-tests_jar-_-any-8837055447507719909/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-14T09:13:38,090 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:35919} 2024-11-14T09:13:38,090 INFO [Time-limited test {}] server.Server(415): Started @4028ms 2024-11-14T09:13:39,100 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T09:13:39,108 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T09:13:39,109 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T09:13:39,109 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T09:13:39,110 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-14T09:13:39,111 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae915b1d-3a0c-f123-b058-6fcd2c888fa8/hadoop.log.dir/,AVAILABLE} 2024-11-14T09:13:39,111 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T09:13:39,234 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae915b1d-3a0c-f123-b058-6fcd2c888fa8/java.io.tmpdir/jetty-localhost-41959-hadoop-hdfs-3_4_1-tests_jar-_-any-12778242878539589302/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T09:13:39,235 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:41959} 2024-11-14T09:13:39,235 INFO [Time-limited test {}] server.Server(415): Started @5173ms 2024-11-14T09:13:39,301 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-14T09:13:40,633 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae915b1d-3a0c-f123-b058-6fcd2c888fa8/cluster_e7e5a44a-d236-23e3-0b04-c3f634c73d33/data/data1/current/BP-784109664-172.17.0.2-1731575615976/current, will proceed with Du for space computation calculation, 2024-11-14T09:13:40,633 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae915b1d-3a0c-f123-b058-6fcd2c888fa8/cluster_e7e5a44a-d236-23e3-0b04-c3f634c73d33/data/data2/current/BP-784109664-172.17.0.2-1731575615976/current, will proceed with Du for space computation calculation, 2024-11-14T09:13:40,668 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-14T09:13:40,734 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf152c26eda2b5597 with lease ID 0x79638973e833fa7a: Processing first storage report for DS-9c7f0760-5409-49ed-93c2-48df037758d2 from datanode DatanodeRegistration(127.0.0.1:37243, datanodeUuid=1fc8a527-2c66-4742-bab0-e6bcd0d07540, infoPort=34887, infoSecurePort=0, ipcPort=41517, storageInfo=lv=-57;cid=testClusterID;nsid=1590416155;c=1731575615976) 2024-11-14T09:13:40,735 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf152c26eda2b5597 with lease ID 0x79638973e833fa7a: from storage DS-9c7f0760-5409-49ed-93c2-48df037758d2 node DatanodeRegistration(127.0.0.1:37243, datanodeUuid=1fc8a527-2c66-4742-bab0-e6bcd0d07540, infoPort=34887, infoSecurePort=0, ipcPort=41517, storageInfo=lv=-57;cid=testClusterID;nsid=1590416155;c=1731575615976), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-14T09:13:40,736 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf152c26eda2b5597 with lease ID 0x79638973e833fa7a: Processing first storage report for DS-aba7bc18-cc21-4cfc-a005-c2a816d072b0 from datanode DatanodeRegistration(127.0.0.1:37243, datanodeUuid=1fc8a527-2c66-4742-bab0-e6bcd0d07540, infoPort=34887, infoSecurePort=0, ipcPort=41517, storageInfo=lv=-57;cid=testClusterID;nsid=1590416155;c=1731575615976) 2024-11-14T09:13:40,736 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf152c26eda2b5597 with lease ID 0x79638973e833fa7a: from storage DS-aba7bc18-cc21-4cfc-a005-c2a816d072b0 node DatanodeRegistration(127.0.0.1:37243, datanodeUuid=1fc8a527-2c66-4742-bab0-e6bcd0d07540, infoPort=34887, infoSecurePort=0, ipcPort=41517, storageInfo=lv=-57;cid=testClusterID;nsid=1590416155;c=1731575615976), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T09:13:40,758 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae915b1d-3a0c-f123-b058-6fcd2c888fa8 2024-11-14T09:13:40,854 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae915b1d-3a0c-f123-b058-6fcd2c888fa8/cluster_e7e5a44a-d236-23e3-0b04-c3f634c73d33/zookeeper_0, clientPort=65171, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae915b1d-3a0c-f123-b058-6fcd2c888fa8/cluster_e7e5a44a-d236-23e3-0b04-c3f634c73d33/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae915b1d-3a0c-f123-b058-6fcd2c888fa8/cluster_e7e5a44a-d236-23e3-0b04-c3f634c73d33/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-14T09:13:40,867 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=65171 2024-11-14T09:13:40,879 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T09:13:40,883 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T09:13:41,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741825_1001 (size=7) 2024-11-14T09:13:41,513 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db with version=8 2024-11-14T09:13:41,513 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/hbase-staging 2024-11-14T09:13:41,606 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-14T09:13:41,854 INFO [Time-limited test {}] client.ConnectionUtils(128): master/83f56b55f2af:0 server-side Connection retries=45 2024-11-14T09:13:41,866 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T09:13:41,867 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T09:13:41,871 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T09:13:41,871 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T09:13:41,872 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T09:13:42,057 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-14T09:13:42,118 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-14T09:13:42,126 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-14T09:13:42,130 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T09:13:42,158 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 3068 (auto-detected) 2024-11-14T09:13:42,159 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-14T09:13:42,179 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45325 2024-11-14T09:13:42,206 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:45325 connecting to ZooKeeper ensemble=127.0.0.1:65171 2024-11-14T09:13:42,239 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:453250x0, quorum=127.0.0.1:65171, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T09:13:42,243 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45325-0x10115c10f700000 connected 2024-11-14T09:13:42,277 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T09:13:42,279 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T09:13:42,288 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45325-0x10115c10f700000, quorum=127.0.0.1:65171, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T09:13:42,292 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db, hbase.cluster.distributed=false 2024-11-14T09:13:42,314 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45325-0x10115c10f700000, quorum=127.0.0.1:65171, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T09:13:42,318 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45325 2024-11-14T09:13:42,319 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45325 2024-11-14T09:13:42,319 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45325 2024-11-14T09:13:42,320 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45325 2024-11-14T09:13:42,320 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45325 2024-11-14T09:13:42,433 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/83f56b55f2af:0 server-side Connection retries=45 2024-11-14T09:13:42,434 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T09:13:42,435 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T09:13:42,435 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T09:13:42,435 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T09:13:42,436 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T09:13:42,439 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-14T09:13:42,441 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T09:13:42,442 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39927 2024-11-14T09:13:42,444 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39927 connecting to ZooKeeper ensemble=127.0.0.1:65171 2024-11-14T09:13:42,445 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T09:13:42,448 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T09:13:42,456 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:399270x0, quorum=127.0.0.1:65171, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T09:13:42,456 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:399270x0, quorum=127.0.0.1:65171, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T09:13:42,456 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39927-0x10115c10f700001 connected 2024-11-14T09:13:42,461 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-14T09:13:42,473 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-14T09:13:42,476 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39927-0x10115c10f700001, quorum=127.0.0.1:65171, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-14T09:13:42,483 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39927-0x10115c10f700001, quorum=127.0.0.1:65171, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T09:13:42,484 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39927 2024-11-14T09:13:42,484 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39927 2024-11-14T09:13:42,485 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39927 2024-11-14T09:13:42,486 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39927 2024-11-14T09:13:42,486 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39927 2024-11-14T09:13:42,510 DEBUG [M:0;83f56b55f2af:45325 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;83f56b55f2af:45325 2024-11-14T09:13:42,511 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/83f56b55f2af,45325,1731575621660 2024-11-14T09:13:42,519 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45325-0x10115c10f700000, quorum=127.0.0.1:65171, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T09:13:42,519 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39927-0x10115c10f700001, quorum=127.0.0.1:65171, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T09:13:42,522 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45325-0x10115c10f700000, quorum=127.0.0.1:65171, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/83f56b55f2af,45325,1731575621660 2024-11-14T09:13:42,545 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45325-0x10115c10f700000, quorum=127.0.0.1:65171, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:13:42,546 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39927-0x10115c10f700001, quorum=127.0.0.1:65171, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-14T09:13:42,546 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39927-0x10115c10f700001, quorum=127.0.0.1:65171, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:13:42,549 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45325-0x10115c10f700000, quorum=127.0.0.1:65171, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-14T09:13:42,550 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/83f56b55f2af,45325,1731575621660 from backup master directory 2024-11-14T09:13:42,554 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39927-0x10115c10f700001, quorum=127.0.0.1:65171, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T09:13:42,554 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45325-0x10115c10f700000, quorum=127.0.0.1:65171, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/83f56b55f2af,45325,1731575621660 2024-11-14T09:13:42,554 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45325-0x10115c10f700000, quorum=127.0.0.1:65171, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T09:13:42,555 WARN [master/83f56b55f2af:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T09:13:42,555 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=83f56b55f2af,45325,1731575621660 2024-11-14T09:13:42,558 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-14T09:13:42,559 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-14T09:13:42,623 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/hbase.id] with ID: 13edf1ec-b933-4893-841a-d61c78de83f6 2024-11-14T09:13:42,624 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/.tmp/hbase.id 2024-11-14T09:13:42,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741826_1002 (size=42) 2024-11-14T09:13:42,646 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/.tmp/hbase.id]:[hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/hbase.id] 2024-11-14T09:13:42,703 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T09:13:42,708 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-14T09:13:42,730 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 20ms. 2024-11-14T09:13:42,735 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39927-0x10115c10f700001, quorum=127.0.0.1:65171, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:13:42,735 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45325-0x10115c10f700000, quorum=127.0.0.1:65171, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:13:42,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741827_1003 (size=196) 2024-11-14T09:13:42,783 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T09:13:42,787 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-14T09:13:42,809 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:42,816 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T09:13:42,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741828_1004 (size=1189) 2024-11-14T09:13:43,296 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/MasterData/data/master/store 2024-11-14T09:13:43,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741829_1005 (size=34) 2024-11-14T09:13:43,723 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-14T09:13:43,727 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:13:43,728 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T09:13:43,728 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T09:13:43,728 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T09:13:43,730 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T09:13:43,730 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T09:13:43,730 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T09:13:43,731 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731575623728Disabling compacts and flushes for region at 1731575623728Disabling writes for close at 1731575623730 (+2 ms)Writing region close event to WAL at 1731575623730Closed at 1731575623730 2024-11-14T09:13:43,733 WARN [master/83f56b55f2af:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/MasterData/data/master/store/.initializing 2024-11-14T09:13:43,734 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/MasterData/WALs/83f56b55f2af,45325,1731575621660 2024-11-14T09:13:43,742 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T09:13:43,758 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=83f56b55f2af%2C45325%2C1731575621660, suffix=, logDir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/MasterData/WALs/83f56b55f2af,45325,1731575621660, archiveDir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/MasterData/oldWALs, maxLogs=10 2024-11-14T09:13:43,782 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/MasterData/WALs/83f56b55f2af,45325,1731575621660/83f56b55f2af%2C45325%2C1731575621660.1731575623764, exclude list is [], retry=0 2024-11-14T09:13:43,798 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37243,DS-9c7f0760-5409-49ed-93c2-48df037758d2,DISK] 2024-11-14T09:13:43,801 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-14T09:13:43,838 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/MasterData/WALs/83f56b55f2af,45325,1731575621660/83f56b55f2af%2C45325%2C1731575621660.1731575623764 2024-11-14T09:13:43,839 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34887:34887)] 2024-11-14T09:13:43,840 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-14T09:13:43,840 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:13:43,845 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T09:13:43,846 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T09:13:43,883 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T09:13:43,910 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-14T09:13:43,913 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:13:43,916 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T09:13:43,916 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T09:13:43,919 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-14T09:13:43,919 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:13:43,921 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:13:43,921 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T09:13:43,923 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-14T09:13:43,923 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:13:43,924 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:13:43,925 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T09:13:43,927 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-14T09:13:43,927 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:13:43,928 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:13:43,928 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T09:13:43,933 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T09:13:43,934 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T09:13:43,940 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T09:13:43,941 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T09:13:43,945 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T09:13:43,949 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T09:13:43,953 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T09:13:43,954 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61451216, jitterRate=-0.0843055248260498}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T09:13:43,962 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731575623859Initializing all the Stores at 1731575623861 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731575623861Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731575623862 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731575623862Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731575623862Cleaning up temporary data from old regions at 1731575623941 (+79 ms)Region opened successfully at 1731575623962 (+21 ms) 2024-11-14T09:13:43,963 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-14T09:13:43,997 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f654f69, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=83f56b55f2af/172.17.0.2:0 2024-11-14T09:13:44,029 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-14T09:13:44,040 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-14T09:13:44,041 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-14T09:13:44,044 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T09:13:44,045 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-14T09:13:44,050 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-14T09:13:44,050 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-14T09:13:44,076 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-14T09:13:44,084 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45325-0x10115c10f700000, quorum=127.0.0.1:65171, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-14T09:13:44,086 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-14T09:13:44,089 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-14T09:13:44,090 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45325-0x10115c10f700000, quorum=127.0.0.1:65171, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-14T09:13:44,092 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-14T09:13:44,094 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-14T09:13:44,097 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45325-0x10115c10f700000, quorum=127.0.0.1:65171, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-14T09:13:44,100 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-14T09:13:44,101 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45325-0x10115c10f700000, quorum=127.0.0.1:65171, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-14T09:13:44,103 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-14T09:13:44,123 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45325-0x10115c10f700000, quorum=127.0.0.1:65171, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-14T09:13:44,126 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-14T09:13:44,130 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45325-0x10115c10f700000, quorum=127.0.0.1:65171, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T09:13:44,130 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39927-0x10115c10f700001, quorum=127.0.0.1:65171, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T09:13:44,131 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45325-0x10115c10f700000, quorum=127.0.0.1:65171, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:13:44,131 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39927-0x10115c10f700001, quorum=127.0.0.1:65171, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:13:44,134 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=83f56b55f2af,45325,1731575621660, sessionid=0x10115c10f700000, setting cluster-up flag (Was=false) 2024-11-14T09:13:44,147 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45325-0x10115c10f700000, quorum=127.0.0.1:65171, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:13:44,147 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39927-0x10115c10f700001, quorum=127.0.0.1:65171, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:13:44,154 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-14T09:13:44,155 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=83f56b55f2af,45325,1731575621660 2024-11-14T09:13:44,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45325-0x10115c10f700000, quorum=127.0.0.1:65171, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:13:44,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39927-0x10115c10f700001, quorum=127.0.0.1:65171, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:13:44,167 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-14T09:13:44,169 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=83f56b55f2af,45325,1731575621660 2024-11-14T09:13:44,175 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-14T09:13:44,191 INFO [RS:0;83f56b55f2af:39927 {}] regionserver.HRegionServer(746): ClusterId : 13edf1ec-b933-4893-841a-d61c78de83f6 2024-11-14T09:13:44,193 DEBUG [RS:0;83f56b55f2af:39927 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-14T09:13:44,198 DEBUG [RS:0;83f56b55f2af:39927 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-14T09:13:44,198 DEBUG [RS:0;83f56b55f2af:39927 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-14T09:13:44,201 DEBUG [RS:0;83f56b55f2af:39927 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-14T09:13:44,201 DEBUG [RS:0;83f56b55f2af:39927 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a65f264, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=83f56b55f2af/172.17.0.2:0 2024-11-14T09:13:44,216 DEBUG [RS:0;83f56b55f2af:39927 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;83f56b55f2af:39927 2024-11-14T09:13:44,221 INFO [RS:0;83f56b55f2af:39927 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-14T09:13:44,221 INFO [RS:0;83f56b55f2af:39927 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-14T09:13:44,221 DEBUG [RS:0;83f56b55f2af:39927 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-14T09:13:44,225 INFO [RS:0;83f56b55f2af:39927 {}] regionserver.HRegionServer(2659): reportForDuty to master=83f56b55f2af,45325,1731575621660 with port=39927, startcode=1731575622393 2024-11-14T09:13:44,238 DEBUG [RS:0;83f56b55f2af:39927 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-14T09:13:44,252 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T09:13:44,263 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-14T09:13:44,272 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-14T09:13:44,276 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43887, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-14T09:13:44,278 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 83f56b55f2af,45325,1731575621660 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-14T09:13:44,284 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45325 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:44,286 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/83f56b55f2af:0, corePoolSize=5, maxPoolSize=5 2024-11-14T09:13:44,286 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/83f56b55f2af:0, corePoolSize=5, maxPoolSize=5 2024-11-14T09:13:44,286 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/83f56b55f2af:0, corePoolSize=5, maxPoolSize=5 2024-11-14T09:13:44,287 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/83f56b55f2af:0, corePoolSize=5, maxPoolSize=5 2024-11-14T09:13:44,287 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/83f56b55f2af:0, corePoolSize=10, maxPoolSize=10 2024-11-14T09:13:44,287 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/83f56b55f2af:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:13:44,287 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/83f56b55f2af:0, corePoolSize=2, maxPoolSize=2 2024-11-14T09:13:44,288 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/83f56b55f2af:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:13:44,292 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731575654292 2024-11-14T09:13:44,293 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T09:13:44,294 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-14T09:13:44,294 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-14T09:13:44,295 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-14T09:13:44,299 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-14T09:13:44,300 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-14T09:13:44,300 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-14T09:13:44,300 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-14T09:13:44,301 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:13:44,302 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-14T09:13:44,301 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T09:13:44,306 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-14T09:13:44,307 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-14T09:13:44,308 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-14T09:13:44,312 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-14T09:13:44,313 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-14T09:13:44,315 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/83f56b55f2af:0:becomeActiveMaster-HFileCleaner.large.0-1731575624314,5,FailOnTimeoutGroup] 2024-11-14T09:13:44,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741831_1007 (size=1321) 2024-11-14T09:13:44,315 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/83f56b55f2af:0:becomeActiveMaster-HFileCleaner.small.0-1731575624315,5,FailOnTimeoutGroup] 2024-11-14T09:13:44,316 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T09:13:44,316 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-14T09:13:44,316 DEBUG [RS:0;83f56b55f2af:39927 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-14T09:13:44,316 WARN [RS:0;83f56b55f2af:39927 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-14T09:13:44,317 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-14T09:13:44,317 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-14T09:13:44,317 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-14T09:13:44,318 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db 2024-11-14T09:13:44,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741832_1008 (size=32) 2024-11-14T09:13:44,418 INFO [RS:0;83f56b55f2af:39927 {}] regionserver.HRegionServer(2659): reportForDuty to master=83f56b55f2af,45325,1731575621660 with port=39927, startcode=1731575622393 2024-11-14T09:13:44,420 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45325 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 83f56b55f2af,39927,1731575622393 2024-11-14T09:13:44,423 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45325 {}] master.ServerManager(517): Registering regionserver=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:44,431 DEBUG [RS:0;83f56b55f2af:39927 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db 2024-11-14T09:13:44,431 DEBUG [RS:0;83f56b55f2af:39927 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44851 2024-11-14T09:13:44,431 DEBUG [RS:0;83f56b55f2af:39927 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-14T09:13:44,437 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45325-0x10115c10f700000, quorum=127.0.0.1:65171, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T09:13:44,437 DEBUG [RS:0;83f56b55f2af:39927 {}] zookeeper.ZKUtil(111): regionserver:39927-0x10115c10f700001, quorum=127.0.0.1:65171, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/83f56b55f2af,39927,1731575622393 2024-11-14T09:13:44,438 WARN [RS:0;83f56b55f2af:39927 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T09:13:44,438 INFO [RS:0;83f56b55f2af:39927 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T09:13:44,438 DEBUG [RS:0;83f56b55f2af:39927 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/WALs/83f56b55f2af,39927,1731575622393 2024-11-14T09:13:44,440 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [83f56b55f2af,39927,1731575622393] 2024-11-14T09:13:44,463 INFO [RS:0;83f56b55f2af:39927 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-14T09:13:44,477 INFO [RS:0;83f56b55f2af:39927 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-14T09:13:44,481 INFO [RS:0;83f56b55f2af:39927 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T09:13:44,482 INFO [RS:0;83f56b55f2af:39927 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T09:13:44,483 INFO [RS:0;83f56b55f2af:39927 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-14T09:13:44,488 INFO [RS:0;83f56b55f2af:39927 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-14T09:13:44,489 INFO [RS:0;83f56b55f2af:39927 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-14T09:13:44,490 DEBUG [RS:0;83f56b55f2af:39927 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/83f56b55f2af:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:13:44,490 DEBUG [RS:0;83f56b55f2af:39927 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/83f56b55f2af:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:13:44,490 DEBUG [RS:0;83f56b55f2af:39927 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/83f56b55f2af:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:13:44,490 DEBUG [RS:0;83f56b55f2af:39927 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/83f56b55f2af:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:13:44,490 DEBUG [RS:0;83f56b55f2af:39927 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/83f56b55f2af:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:13:44,490 DEBUG [RS:0;83f56b55f2af:39927 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/83f56b55f2af:0, corePoolSize=2, maxPoolSize=2 2024-11-14T09:13:44,491 DEBUG [RS:0;83f56b55f2af:39927 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/83f56b55f2af:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:13:44,491 DEBUG [RS:0;83f56b55f2af:39927 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/83f56b55f2af:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:13:44,491 DEBUG [RS:0;83f56b55f2af:39927 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/83f56b55f2af:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:13:44,491 DEBUG [RS:0;83f56b55f2af:39927 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/83f56b55f2af:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:13:44,491 DEBUG [RS:0;83f56b55f2af:39927 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/83f56b55f2af:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:13:44,491 DEBUG [RS:0;83f56b55f2af:39927 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/83f56b55f2af:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:13:44,491 DEBUG [RS:0;83f56b55f2af:39927 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/83f56b55f2af:0, corePoolSize=3, maxPoolSize=3 2024-11-14T09:13:44,492 DEBUG [RS:0;83f56b55f2af:39927 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0, corePoolSize=3, maxPoolSize=3 2024-11-14T09:13:44,492 INFO [RS:0;83f56b55f2af:39927 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T09:13:44,493 INFO [RS:0;83f56b55f2af:39927 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T09:13:44,493 INFO [RS:0;83f56b55f2af:39927 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T09:13:44,493 INFO [RS:0;83f56b55f2af:39927 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-14T09:13:44,493 INFO [RS:0;83f56b55f2af:39927 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-14T09:13:44,494 INFO [RS:0;83f56b55f2af:39927 {}] hbase.ChoreService(168): Chore ScheduledChore name=83f56b55f2af,39927,1731575622393-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T09:13:44,514 INFO [RS:0;83f56b55f2af:39927 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-14T09:13:44,515 INFO [RS:0;83f56b55f2af:39927 {}] hbase.ChoreService(168): Chore ScheduledChore name=83f56b55f2af,39927,1731575622393-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T09:13:44,516 INFO [RS:0;83f56b55f2af:39927 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T09:13:44,516 INFO [RS:0;83f56b55f2af:39927 {}] regionserver.Replication(171): 83f56b55f2af,39927,1731575622393 started 2024-11-14T09:13:44,535 INFO [RS:0;83f56b55f2af:39927 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T09:13:44,535 INFO [RS:0;83f56b55f2af:39927 {}] regionserver.HRegionServer(1482): Serving as 83f56b55f2af,39927,1731575622393, RpcServer on 83f56b55f2af/172.17.0.2:39927, sessionid=0x10115c10f700001 2024-11-14T09:13:44,536 DEBUG [RS:0;83f56b55f2af:39927 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-14T09:13:44,536 DEBUG [RS:0;83f56b55f2af:39927 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 83f56b55f2af,39927,1731575622393 2024-11-14T09:13:44,536 DEBUG [RS:0;83f56b55f2af:39927 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '83f56b55f2af,39927,1731575622393' 2024-11-14T09:13:44,536 DEBUG [RS:0;83f56b55f2af:39927 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-14T09:13:44,538 DEBUG [RS:0;83f56b55f2af:39927 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-14T09:13:44,538 DEBUG [RS:0;83f56b55f2af:39927 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-14T09:13:44,539 DEBUG [RS:0;83f56b55f2af:39927 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-14T09:13:44,539 DEBUG [RS:0;83f56b55f2af:39927 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 83f56b55f2af,39927,1731575622393 2024-11-14T09:13:44,539 DEBUG [RS:0;83f56b55f2af:39927 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '83f56b55f2af,39927,1731575622393' 2024-11-14T09:13:44,539 DEBUG [RS:0;83f56b55f2af:39927 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-14T09:13:44,540 DEBUG [RS:0;83f56b55f2af:39927 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-14T09:13:44,541 DEBUG [RS:0;83f56b55f2af:39927 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-14T09:13:44,541 INFO [RS:0;83f56b55f2af:39927 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-14T09:13:44,541 INFO [RS:0;83f56b55f2af:39927 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-14T09:13:44,647 INFO [RS:0;83f56b55f2af:39927 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T09:13:44,650 INFO [RS:0;83f56b55f2af:39927 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=83f56b55f2af%2C39927%2C1731575622393, suffix=, logDir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/WALs/83f56b55f2af,39927,1731575622393, archiveDir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/oldWALs, maxLogs=32 2024-11-14T09:13:44,668 DEBUG [RS:0;83f56b55f2af:39927 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/WALs/83f56b55f2af,39927,1731575622393/83f56b55f2af%2C39927%2C1731575622393.1731575624653, exclude list is [], retry=0 2024-11-14T09:13:44,673 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37243,DS-9c7f0760-5409-49ed-93c2-48df037758d2,DISK] 2024-11-14T09:13:44,676 INFO [RS:0;83f56b55f2af:39927 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/WALs/83f56b55f2af,39927,1731575622393/83f56b55f2af%2C39927%2C1731575622393.1731575624653 2024-11-14T09:13:44,676 DEBUG [RS:0;83f56b55f2af:39927 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34887:34887)] 2024-11-14T09:13:44,729 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:13:44,732 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-14T09:13:44,735 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T09:13:44,735 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:13:44,736 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T09:13:44,736 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-14T09:13:44,738 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T09:13:44,738 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:13:44,739 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T09:13:44,739 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-14T09:13:44,742 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T09:13:44,742 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:13:44,743 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T09:13:44,743 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-14T09:13:44,745 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T09:13:44,745 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:13:44,746 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T09:13:44,746 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T09:13:44,747 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/hbase/meta/1588230740 2024-11-14T09:13:44,748 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/hbase/meta/1588230740 2024-11-14T09:13:44,751 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T09:13:44,751 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T09:13:44,752 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T09:13:44,754 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T09:13:44,757 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T09:13:44,758 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74801262, jitterRate=0.11462566256523132}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T09:13:44,760 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731575624729Initializing all the Stores at 1731575624731 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731575624732 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731575624732Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731575624732Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731575624732Cleaning up temporary data from old regions at 1731575624751 (+19 ms)Region opened successfully at 1731575624760 (+9 ms) 2024-11-14T09:13:44,760 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T09:13:44,761 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T09:13:44,761 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T09:13:44,761 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T09:13:44,761 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T09:13:44,762 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T09:13:44,762 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731575624760Disabling compacts and flushes for region at 1731575624760Disabling writes for close at 1731575624761 (+1 ms)Writing region close event to WAL at 1731575624762 (+1 ms)Closed at 1731575624762 2024-11-14T09:13:44,766 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T09:13:44,766 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-14T09:13:44,773 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-14T09:13:44,781 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T09:13:44,784 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-14T09:13:44,936 DEBUG [83f56b55f2af:45325 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-14T09:13:44,948 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:44,956 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 83f56b55f2af,39927,1731575622393, state=OPENING 2024-11-14T09:13:44,961 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-14T09:13:44,963 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45325-0x10115c10f700000, quorum=127.0.0.1:65171, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:13:44,963 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39927-0x10115c10f700001, quorum=127.0.0.1:65171, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:13:44,964 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T09:13:44,964 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T09:13:44,966 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T09:13:44,967 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=83f56b55f2af,39927,1731575622393}] 2024-11-14T09:13:45,143 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-14T09:13:45,147 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43127, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-14T09:13:45,158 INFO [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-14T09:13:45,158 INFO [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T09:13:45,159 INFO [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-14T09:13:45,162 INFO [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=83f56b55f2af%2C39927%2C1731575622393.meta, suffix=.meta, logDir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/WALs/83f56b55f2af,39927,1731575622393, archiveDir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/oldWALs, maxLogs=32 2024-11-14T09:13:45,179 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/WALs/83f56b55f2af,39927,1731575622393/83f56b55f2af%2C39927%2C1731575622393.meta.1731575625165.meta, exclude list is [], retry=0 2024-11-14T09:13:45,184 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37243,DS-9c7f0760-5409-49ed-93c2-48df037758d2,DISK] 2024-11-14T09:13:45,187 INFO [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/WALs/83f56b55f2af,39927,1731575622393/83f56b55f2af%2C39927%2C1731575622393.meta.1731575625165.meta 2024-11-14T09:13:45,187 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34887:34887)] 2024-11-14T09:13:45,188 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-14T09:13:45,190 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-14T09:13:45,193 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-14T09:13:45,198 INFO [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-14T09:13:45,203 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-14T09:13:45,203 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:13:45,204 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-14T09:13:45,204 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-14T09:13:45,208 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-14T09:13:45,209 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T09:13:45,209 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:13:45,210 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T09:13:45,211 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-14T09:13:45,212 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T09:13:45,212 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:13:45,213 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T09:13:45,213 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-14T09:13:45,214 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T09:13:45,215 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:13:45,215 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T09:13:45,216 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-14T09:13:45,217 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T09:13:45,217 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:13:45,218 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T09:13:45,218 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T09:13:45,220 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/hbase/meta/1588230740 2024-11-14T09:13:45,223 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/hbase/meta/1588230740 2024-11-14T09:13:45,225 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T09:13:45,225 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T09:13:45,226 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T09:13:45,229 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T09:13:45,231 INFO [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59750432, jitterRate=-0.1096491813659668}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T09:13:45,231 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-14T09:13:45,233 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731575625205Writing region info on filesystem at 1731575625205Initializing all the Stores at 1731575625207 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731575625207Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731575625207Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731575625208 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731575625208Cleaning up temporary data from old regions at 1731575625225 (+17 ms)Running coprocessor post-open hooks at 1731575625231 (+6 ms)Region opened successfully at 1731575625233 (+2 ms) 2024-11-14T09:13:45,241 INFO [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731575625134 2024-11-14T09:13:45,253 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-14T09:13:45,254 INFO [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-14T09:13:45,256 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:45,258 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 83f56b55f2af,39927,1731575622393, state=OPEN 2024-11-14T09:13:45,264 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45325-0x10115c10f700000, quorum=127.0.0.1:65171, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T09:13:45,264 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39927-0x10115c10f700001, quorum=127.0.0.1:65171, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T09:13:45,264 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T09:13:45,264 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T09:13:45,264 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:45,271 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-14T09:13:45,271 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=83f56b55f2af,39927,1731575622393 in 297 msec 2024-11-14T09:13:45,279 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-14T09:13:45,279 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 502 msec 2024-11-14T09:13:45,280 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T09:13:45,281 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-14T09:13:45,302 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:13:45,303 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:13:45,326 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:13:45,328 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43333, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:13:45,350 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1430 sec 2024-11-14T09:13:45,350 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731575625350, completionTime=-1 2024-11-14T09:13:45,354 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-14T09:13:45,354 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-14T09:13:45,384 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-14T09:13:45,384 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731575685384 2024-11-14T09:13:45,385 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731575745385 2024-11-14T09:13:45,385 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 30 msec 2024-11-14T09:13:45,387 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=83f56b55f2af,45325,1731575621660-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T09:13:45,387 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=83f56b55f2af,45325,1731575621660-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T09:13:45,387 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=83f56b55f2af,45325,1731575621660-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T09:13:45,389 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-83f56b55f2af:45325, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T09:13:45,390 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-14T09:13:45,390 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-14T09:13:45,396 DEBUG [master/83f56b55f2af:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-14T09:13:45,418 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.863sec 2024-11-14T09:13:45,420 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-14T09:13:45,421 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-14T09:13:45,422 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-14T09:13:45,423 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-14T09:13:45,423 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-14T09:13:45,424 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=83f56b55f2af,45325,1731575621660-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T09:13:45,424 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=83f56b55f2af,45325,1731575621660-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-14T09:13:45,433 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-14T09:13:45,434 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-14T09:13:45,434 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=83f56b55f2af,45325,1731575621660-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T09:13:45,504 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:13:45,507 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-14T09:13:45,507 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-14T09:13:45,511 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:13:45,515 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:13:45,523 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:13:45,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:13:45,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:13:45,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15e8df74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:13:45,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:13:45,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:13:45,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:13:45,537 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40252, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:13:45,540 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:13:45,541 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:13:45,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:13:45,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:13:45,553 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49926, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:13:45,575 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=83f56b55f2af,45325,1731575621660 2024-11-14T09:13:45,590 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=279, ProcessCount=11, AvailableMemoryMB=7344 2024-11-14T09:13:45,600 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T09:13:45,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 83f56b55f2af,45325,1731575621660 2024-11-14T09:13:45,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@216dfbe8 2024-11-14T09:13:45,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T09:13:45,638 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40264, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T09:13:45,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45325 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T09:13:45,645 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45325 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T09:13:45,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T09:13:45,662 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T09:13:45,665 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45325 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-14T09:13:45,665 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:13:45,668 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T09:13:45,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T09:13:45,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741835_1011 (size=1061) 2024-11-14T09:13:45,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T09:13:45,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T09:13:46,085 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db 2024-11-14T09:13:46,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741836_1012 (size=53) 2024-11-14T09:13:46,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T09:13:46,504 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:13:46,505 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing a92123abacf02fce4f65c3ba94256e83, disabling compactions & flushes 2024-11-14T09:13:46,505 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:46,505 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:46,505 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. after waiting 0 ms 2024-11-14T09:13:46,505 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:46,505 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:46,505 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for a92123abacf02fce4f65c3ba94256e83: Waiting for close lock at 1731575626505Disabling compacts and flushes for region at 1731575626505Disabling writes for close at 1731575626505Writing region close event to WAL at 1731575626505Closed at 1731575626505 2024-11-14T09:13:46,508 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T09:13:46,515 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731575626509"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731575626509"}]},"ts":"1731575626509"} 2024-11-14T09:13:46,522 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T09:13:46,524 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T09:13:46,528 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731575626525"}]},"ts":"1731575626525"} 2024-11-14T09:13:46,534 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T09:13:46,536 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a92123abacf02fce4f65c3ba94256e83, ASSIGN}] 2024-11-14T09:13:46,539 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a92123abacf02fce4f65c3ba94256e83, ASSIGN 2024-11-14T09:13:46,541 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a92123abacf02fce4f65c3ba94256e83, ASSIGN; state=OFFLINE, location=83f56b55f2af,39927,1731575622393; forceNewPlan=false, retain=false 2024-11-14T09:13:46,693 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=a92123abacf02fce4f65c3ba94256e83, regionState=OPENING, regionLocation=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:46,698 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a92123abacf02fce4f65c3ba94256e83, ASSIGN because future has completed 2024-11-14T09:13:46,699 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393}] 2024-11-14T09:13:46,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T09:13:46,865 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:46,865 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} 2024-11-14T09:13:46,866 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a92123abacf02fce4f65c3ba94256e83 2024-11-14T09:13:46,866 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:13:46,866 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for a92123abacf02fce4f65c3ba94256e83 2024-11-14T09:13:46,866 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for a92123abacf02fce4f65c3ba94256e83 2024-11-14T09:13:46,869 INFO [StoreOpener-a92123abacf02fce4f65c3ba94256e83-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a92123abacf02fce4f65c3ba94256e83 2024-11-14T09:13:46,874 INFO [StoreOpener-a92123abacf02fce4f65c3ba94256e83-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T09:13:46,875 INFO [StoreOpener-a92123abacf02fce4f65c3ba94256e83-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a92123abacf02fce4f65c3ba94256e83 columnFamilyName A 2024-11-14T09:13:46,875 DEBUG [StoreOpener-a92123abacf02fce4f65c3ba94256e83-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:13:46,876 INFO [StoreOpener-a92123abacf02fce4f65c3ba94256e83-1 {}] regionserver.HStore(327): Store=a92123abacf02fce4f65c3ba94256e83/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:13:46,877 INFO [StoreOpener-a92123abacf02fce4f65c3ba94256e83-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a92123abacf02fce4f65c3ba94256e83 2024-11-14T09:13:46,879 INFO [StoreOpener-a92123abacf02fce4f65c3ba94256e83-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T09:13:46,880 INFO [StoreOpener-a92123abacf02fce4f65c3ba94256e83-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a92123abacf02fce4f65c3ba94256e83 columnFamilyName B 2024-11-14T09:13:46,880 DEBUG [StoreOpener-a92123abacf02fce4f65c3ba94256e83-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:13:46,881 INFO [StoreOpener-a92123abacf02fce4f65c3ba94256e83-1 {}] regionserver.HStore(327): Store=a92123abacf02fce4f65c3ba94256e83/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:13:46,881 INFO [StoreOpener-a92123abacf02fce4f65c3ba94256e83-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a92123abacf02fce4f65c3ba94256e83 2024-11-14T09:13:46,884 INFO [StoreOpener-a92123abacf02fce4f65c3ba94256e83-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T09:13:46,884 INFO [StoreOpener-a92123abacf02fce4f65c3ba94256e83-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a92123abacf02fce4f65c3ba94256e83 columnFamilyName C 2024-11-14T09:13:46,885 DEBUG [StoreOpener-a92123abacf02fce4f65c3ba94256e83-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:13:46,885 INFO [StoreOpener-a92123abacf02fce4f65c3ba94256e83-1 {}] regionserver.HStore(327): Store=a92123abacf02fce4f65c3ba94256e83/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:13:46,886 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:46,886 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for a92123abacf02fce4f65c3ba94256e83 2024-11-14T09:13:46,888 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83 2024-11-14T09:13:46,889 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83 2024-11-14T09:13:46,892 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for a92123abacf02fce4f65c3ba94256e83 2024-11-14T09:13:46,892 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for a92123abacf02fce4f65c3ba94256e83 2024-11-14T09:13:46,893 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T09:13:46,897 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for a92123abacf02fce4f65c3ba94256e83 2024-11-14T09:13:46,906 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T09:13:46,907 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened a92123abacf02fce4f65c3ba94256e83; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64215555, jitterRate=-0.04311366379261017}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T09:13:46,908 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a92123abacf02fce4f65c3ba94256e83 2024-11-14T09:13:46,909 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for a92123abacf02fce4f65c3ba94256e83: Running coprocessor pre-open hook at 1731575626866Writing region info on filesystem at 1731575626866Initializing all the Stores at 1731575626869 (+3 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731575626869Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731575626869Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731575626869Cleaning up temporary data from old regions at 1731575626892 (+23 ms)Running coprocessor post-open hooks at 1731575626908 (+16 ms)Region opened successfully at 1731575626909 (+1 ms) 2024-11-14T09:13:46,913 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., pid=6, masterSystemTime=1731575626857 2024-11-14T09:13:46,919 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:46,919 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:46,920 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=a92123abacf02fce4f65c3ba94256e83, regionState=OPEN, openSeqNum=2, regionLocation=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:46,925 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 because future has completed 2024-11-14T09:13:46,938 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-14T09:13:46,938 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 in 233 msec 2024-11-14T09:13:46,946 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-14T09:13:46,946 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a92123abacf02fce4f65c3ba94256e83, ASSIGN in 403 msec 2024-11-14T09:13:46,948 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T09:13:46,948 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731575626948"}]},"ts":"1731575626948"} 2024-11-14T09:13:46,954 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T09:13:46,956 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T09:13:46,961 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.3070 sec 2024-11-14T09:13:47,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T09:13:47,817 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:13:47,822 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:13:47,822 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:13:47,822 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:13:47,824 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:13:47,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:13:47,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:13:47,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c16416b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:13:47,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:13:47,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:13:47,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:13:47,827 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40290, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:13:47,829 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f9a777e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:13:47,829 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:13:47,831 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:13:47,831 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:13:47,834 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49934, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:13:47,836 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T09:13:47,837 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 83f56b55f2af,45325,1731575621660 2024-11-14T09:13:47,837 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@144e408 2024-11-14T09:13:47,840 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T09:13:47,842 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40298, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T09:13:47,846 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:13:47,846 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:13:47,846 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:13:47,848 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:13:47,848 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:13:47,848 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:13:47,849 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@175526a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:13:47,849 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:13:47,849 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:13:47,849 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:13:47,851 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40314, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:13:47,852 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f55d533, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:13:47,853 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:13:47,853 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:13:47,853 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:13:47,856 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:13:47,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:13:47,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:13:47,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ccb4eda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:13:47,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:13:47,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:13:47,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:13:47,859 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40322, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:13:47,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fc8436a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:13:47,861 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:13:47,861 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:13:47,861 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:13:47,862 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:13:47,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:13:47,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:13:47,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10bc6ecb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:13:47,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:13:47,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:13:47,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:13:47,865 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40332, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:13:47,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64257f01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:13:47,866 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:13:47,866 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:13:47,866 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:13:47,868 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:13:47,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:13:47,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:13:47,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a27ead7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:13:47,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:13:47,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:13:47,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:13:47,870 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40354, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:13:47,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@457a3094, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:13:47,872 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:13:47,872 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:13:47,872 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:13:47,873 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:13:47,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:13:47,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:13:47,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@779a286f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:13:47,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:13:47,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:13:47,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:13:47,875 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40378, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:13:47,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54571dc0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:13:47,877 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:13:47,877 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:13:47,877 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:13:47,878 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:13:47,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:13:47,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:13:47,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62fad981, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:13:47,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:13:47,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:13:47,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:13:47,880 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40392, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:13:47,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43cb8306, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:13:47,882 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:13:47,882 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:13:47,882 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:13:47,883 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:13:47,883 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:13:47,883 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:13:47,883 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79201cc2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:13:47,883 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:13:47,884 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:13:47,884 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:13:47,885 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40414, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:13:47,885 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a98f2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:13:47,886 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:13:47,886 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:13:47,887 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:13:47,887 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:13:47,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:13:47,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:13:47,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12f232f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:13:47,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:13:47,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:13:47,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:13:47,889 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40442, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:13:47,890 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5be3f202, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:13:47,891 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:13:47,891 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:13:47,891 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:13:47,891 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:13:47,892 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:13:47,892 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:13:47,892 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21f437cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:13:47,892 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:13:47,892 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:13:47,892 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:13:47,893 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40458, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:13:47,894 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a903853, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:13:47,898 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:13:47,898 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:13:47,898 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:13:47,898 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:13:47,899 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:13:47,900 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:13:47,900 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:13:47,900 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:13:47,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:13:47,900 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:13:47,900 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:13:47,901 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:13:47,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:13:47,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:13:47,902 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:13:47,903 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:13:47,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:13:47,903 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:13:47,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:13:47,903 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49936, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:13:47,903 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49938, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:13:47,903 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49950, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:13:47,904 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:13:47,905 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:13:47,905 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49960, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:13:47,906 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49966, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:13:47,906 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:13:47,906 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49994, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:13:47,906 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49980, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:13:47,907 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:13:47,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:13:47,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:13:47,910 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49996, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:13:47,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:13:47,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:13:47,912 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50002, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:13:47,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2] 2024-11-14T09:13:47,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2] 2024-11-14T09:13:47,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2] 2024-11-14T09:13:47,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2] 2024-11-14T09:13:47,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2] 2024-11-14T09:13:47,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2] 2024-11-14T09:13:47,922 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2] 2024-11-14T09:13:47,924 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:13:47,925 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2] 2024-11-14T09:13:47,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2] 2024-11-14T09:13:47,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-14T09:13:47,933 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:13:47,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T09:13:47,936 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:13:47,938 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:13:47,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on a92123abacf02fce4f65c3ba94256e83 2024-11-14T09:13:47,989 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a92123abacf02fce4f65c3ba94256e83 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T09:13:48,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=A 2024-11-14T09:13:48,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:13:48,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=B 2024-11-14T09:13:48,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:13:48,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=C 2024-11-14T09:13:48,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:13:48,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T09:13:48,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:48,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T09:13:48,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49980 deadline: 1731575688091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:48,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:48,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49996 deadline: 1731575688098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:48,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:48,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:48,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:48,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:48,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:48,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:48,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:48,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575688099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:48,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:48,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:48,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:48,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:48,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:13:48,121 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/48e3bdfc754c47d9a827e09d90ec10e5 is 50, key is test_row_0/A:col10/1731575627986/Put/seqid=0 2024-11-14T09:13:48,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:48,122 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:48,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:48,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:48,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:48,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575688103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:48,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:48,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:48,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:48,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741837_1013 (size=14341) 2024-11-14T09:13:48,143 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/48e3bdfc754c47d9a827e09d90ec10e5 2024-11-14T09:13:48,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:48,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49966 deadline: 1731575688115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:48,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:48,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:48,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:48,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:48,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49996 deadline: 1731575688216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:48,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:48,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:48,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:48,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:48,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575688217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:48,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:48,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:48,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:48,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:48,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49980 deadline: 1731575688217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:48,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:48,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:48,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:48,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:48,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575688245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:48,248 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:48,248 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:48,248 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:48,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T09:13:48,261 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/9a5626a483f944d494ac17692769a729 is 50, key is test_row_0/B:col10/1731575627986/Put/seqid=0 2024-11-14T09:13:48,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:48,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49966 deadline: 1731575688256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:48,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:48,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:48,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:48,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741838_1014 (size=12001) 2024-11-14T09:13:48,288 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/9a5626a483f944d494ac17692769a729 2024-11-14T09:13:48,288 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T09:13:48,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:48,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:13:48,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:48,300 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:48,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:48,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:48,329 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/ac7dc2dcde354dedb5bf0b92634c7b9a is 50, key is test_row_0/C:col10/1731575627986/Put/seqid=0 2024-11-14T09:13:48,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741839_1015 (size=12001) 2024-11-14T09:13:48,352 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/ac7dc2dcde354dedb5bf0b92634c7b9a 2024-11-14T09:13:48,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/48e3bdfc754c47d9a827e09d90ec10e5 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/48e3bdfc754c47d9a827e09d90ec10e5 2024-11-14T09:13:48,383 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/48e3bdfc754c47d9a827e09d90ec10e5, entries=200, sequenceid=14, filesize=14.0 K 2024-11-14T09:13:48,389 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-14T09:13:48,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/9a5626a483f944d494ac17692769a729 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/9a5626a483f944d494ac17692769a729 2024-11-14T09:13:48,407 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/9a5626a483f944d494ac17692769a729, entries=150, sequenceid=14, filesize=11.7 K 2024-11-14T09:13:48,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/ac7dc2dcde354dedb5bf0b92634c7b9a as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/ac7dc2dcde354dedb5bf0b92634c7b9a 2024-11-14T09:13:48,436 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/ac7dc2dcde354dedb5bf0b92634c7b9a, entries=150, sequenceid=14, filesize=11.7 K 2024-11-14T09:13:48,444 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for a92123abacf02fce4f65c3ba94256e83 in 453ms, sequenceid=14, compaction requested=false 2024-11-14T09:13:48,446 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-14T09:13:48,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a92123abacf02fce4f65c3ba94256e83: 2024-11-14T09:13:48,454 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T09:13:48,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on a92123abacf02fce4f65c3ba94256e83 2024-11-14T09:13:48,461 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a92123abacf02fce4f65c3ba94256e83 3/3 column families, dataSize=127.47 KB heapSize=379.97 KB 2024-11-14T09:13:48,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=A 2024-11-14T09:13:48,463 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731575628086, max timestamp=1731575628445Num uniques 859; 2024-11-14T09:13:48,465 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731575628086, max timestamp=1731575628445Num uniques 859; 2024-11-14T09:13:48,465 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:13:48,465 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731575628086, max timestamp=1731575628445Num uniques 859; 2024-11-14T09:13:48,466 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:13:48,472 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:13:48,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:13:48,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=B 2024-11-14T09:13:48,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:13:48,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=C 2024-11-14T09:13:48,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:13:48,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:48,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:13:48,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:48,481 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:48,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:48,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:48,493 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/c309c9d7673144ec8c135c77844a4b17 is 50, key is test_row_0/A:col10/1731575628097/Put/seqid=0 2024-11-14T09:13:48,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741840_1016 (size=16681) 2024-11-14T09:13:48,521 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/c309c9d7673144ec8c135c77844a4b17 2024-11-14T09:13:48,545 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/e7cacf86dde84ea0ae65d6fb0d70417b is 50, key is test_row_0/B:col10/1731575628097/Put/seqid=0 2024-11-14T09:13:48,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T09:13:48,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741841_1017 (size=12001) 2024-11-14T09:13:48,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731575628544, max timestamp=1731575628574Num uniques 859; 2024-11-14T09:13:48,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:13:48,597 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731575628544, max timestamp=1731575628579Num uniques 859; 2024-11-14T09:13:48,598 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:13:48,599 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731575628544, max timestamp=1731575628574Num uniques 859; 2024-11-14T09:13:48,599 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:13:48,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T09:13:48,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:48,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:13:48,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:48,649 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:48,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:48,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:48,677 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=183, cellSize=8378, totalHeapSize=19272, min timestamp=1731575628622, max timestamp=1731575628659Num uniques 1018; 2024-11-14T09:13:48,677 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=8378, suffix heap size=133840,new segment heap size=19272 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=183 2024-11-14T09:13:48,681 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731575628622, max timestamp=1731575628663Num uniques 1018; 2024-11-14T09:13:48,682 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-14T09:13:48,683 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731575628622, max timestamp=1731575628664Num uniques 1018; 2024-11-14T09:13:48,683 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-14T09:13:48,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:48,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575688732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:48,758 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731575628721, max timestamp=1731575628751Num uniques 1027; 2024-11-14T09:13:48,758 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731575628721, max timestamp=1731575628751Num uniques 1027; 2024-11-14T09:13:48,758 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:48,758 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-14T09:13:48,758 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:48,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:48,758 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-14T09:13:48,765 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731575628721, max timestamp=1731575628747Num uniques 1042; 2024-11-14T09:13:48,766 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47722, new segment data size=6870, suffix heap size=136336,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1042, new segment cells count=150 2024-11-14T09:13:48,806 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T09:13:48,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:48,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:13:48,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:48,807 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:48,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:48,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:48,885 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1731575628859, max timestamp=1731575628880Num uniques 1009; 2024-11-14T09:13:48,886 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=15386, suffix heap size=132904,new segment heap size=35184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=336 2024-11-14T09:13:48,887 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1731575628859, max timestamp=1731575628880Num uniques 1086; 2024-11-14T09:13:48,887 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1731575628859, max timestamp=1731575628880Num uniques 1086; 2024-11-14T09:13:48,887 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=15386, suffix heap size=140912,new segment heap size=35184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=336 2024-11-14T09:13:48,887 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=15386, suffix heap size=140912,new segment heap size=35184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=336 2024-11-14T09:13:48,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:48,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575688947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:48,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:48,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:48,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:48,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49980 deadline: 1731575688948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:48,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:48,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:48,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:48,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:48,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:48,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:48,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T09:13:48,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49966 deadline: 1731575688952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:48,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575688952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:48,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:48,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:13:48,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:48,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:48,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:48,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49996 deadline: 1731575688954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:48,964 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:48,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:48,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:48,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:48,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:48,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:48,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:48,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:48,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:48,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:48,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:48,970 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/e7cacf86dde84ea0ae65d6fb0d70417b 2024-11-14T09:13:48,989 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/9131cf1925354ac0b1a2c87dfa49ef14 is 50, key is test_row_0/C:col10/1731575628097/Put/seqid=0 2024-11-14T09:13:49,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741842_1018 (size=12001) 2024-11-14T09:13:49,035 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/9131cf1925354ac0b1a2c87dfa49ef14 2024-11-14T09:13:49,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/c309c9d7673144ec8c135c77844a4b17 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/c309c9d7673144ec8c135c77844a4b17 2024-11-14T09:13:49,060 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/c309c9d7673144ec8c135c77844a4b17, entries=250, sequenceid=36, filesize=16.3 K 2024-11-14T09:13:49,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/e7cacf86dde84ea0ae65d6fb0d70417b as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/e7cacf86dde84ea0ae65d6fb0d70417b 2024-11-14T09:13:49,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:49,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575689066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:49,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:49,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:49,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:49,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:49,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49980 deadline: 1731575689066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:49,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:49,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:49,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:49,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T09:13:49,080 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/e7cacf86dde84ea0ae65d6fb0d70417b, entries=150, sequenceid=36, filesize=11.7 K 2024-11-14T09:13:49,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:49,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49966 deadline: 1731575689076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:49,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/9131cf1925354ac0b1a2c87dfa49ef14 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/9131cf1925354ac0b1a2c87dfa49ef14 2024-11-14T09:13:49,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:49,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:49,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:49,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:49,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:49,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49996 deadline: 1731575689077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:49,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575689077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:49,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:49,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:49,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:49,086 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:49,086 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:49,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:49,097 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/9131cf1925354ac0b1a2c87dfa49ef14, entries=150, sequenceid=36, filesize=11.7 K 2024-11-14T09:13:49,099 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=154.31 KB/158010 for a92123abacf02fce4f65c3ba94256e83 in 638ms, sequenceid=36, compaction requested=false 2024-11-14T09:13:49,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a92123abacf02fce4f65c3ba94256e83: 2024-11-14T09:13:49,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T09:13:49,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:49,121 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing a92123abacf02fce4f65c3ba94256e83 3/3 column families, dataSize=154.31 KB heapSize=428.16 KB 2024-11-14T09:13:49,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=A 2024-11-14T09:13:49,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:13:49,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=B 2024-11-14T09:13:49,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:13:49,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=C 2024-11-14T09:13:49,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:13:49,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/9ab6d0155d964562a5cd5b4716cc8bff is 50, key is test_row_0/A:col10/1731575628939/Put/seqid=0 2024-11-14T09:13:49,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741843_1019 (size=12001) 2024-11-14T09:13:49,161 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/9ab6d0155d964562a5cd5b4716cc8bff 2024-11-14T09:13:49,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/3e17093d314248e49c7cc0f738ad3e5f is 50, key is test_row_0/B:col10/1731575628939/Put/seqid=0 2024-11-14T09:13:49,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741844_1020 (size=12001) 2024-11-14T09:13:49,228 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/3e17093d314248e49c7cc0f738ad3e5f 2024-11-14T09:13:49,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/fe0d594cf40d467d9b9b939c49abfcf4 is 50, key is test_row_0/C:col10/1731575628939/Put/seqid=0 2024-11-14T09:13:49,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:13:49,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on a92123abacf02fce4f65c3ba94256e83 2024-11-14T09:13:49,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741845_1021 (size=12001) 2024-11-14T09:13:49,296 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/fe0d594cf40d467d9b9b939c49abfcf4 2024-11-14T09:13:49,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:49,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575689297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:49,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:49,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:49,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:49,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:49,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49996 deadline: 1731575689298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:49,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:49,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/9ab6d0155d964562a5cd5b4716cc8bff as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/9ab6d0155d964562a5cd5b4716cc8bff 2024-11-14T09:13:49,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:49,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:49,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:49,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575689298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:49,312 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:49,312 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:49,312 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:49,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:49,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49966 deadline: 1731575689297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:49,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:49,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:49,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:49,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:49,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49980 deadline: 1731575689306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:49,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:49,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:49,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:49,338 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/9ab6d0155d964562a5cd5b4716cc8bff, entries=150, sequenceid=124, filesize=11.7 K 2024-11-14T09:13:49,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/3e17093d314248e49c7cc0f738ad3e5f as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/3e17093d314248e49c7cc0f738ad3e5f 2024-11-14T09:13:49,354 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/3e17093d314248e49c7cc0f738ad3e5f, entries=150, sequenceid=124, filesize=11.7 K 2024-11-14T09:13:49,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/fe0d594cf40d467d9b9b939c49abfcf4 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/fe0d594cf40d467d9b9b939c49abfcf4 2024-11-14T09:13:49,371 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/fe0d594cf40d467d9b9b939c49abfcf4, entries=150, sequenceid=124, filesize=11.7 K 2024-11-14T09:13:49,372 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~428.11 KB/438384, currentSize=33.54 KB/34350 for a92123abacf02fce4f65c3ba94256e83 in 252ms, sequenceid=124, compaction requested=true 2024-11-14T09:13:49,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for a92123abacf02fce4f65c3ba94256e83: 2024-11-14T09:13:49,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:49,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-14T09:13:49,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-14T09:13:49,379 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-14T09:13:49,379 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4370 sec 2024-11-14T09:13:49,385 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.4540 sec 2024-11-14T09:13:49,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on a92123abacf02fce4f65c3ba94256e83 2024-11-14T09:13:49,429 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a92123abacf02fce4f65c3ba94256e83 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T09:13:49,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=A 2024-11-14T09:13:49,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:13:49,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=B 2024-11-14T09:13:49,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:13:49,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=C 2024-11-14T09:13:49,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:13:49,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/4f384c57c2474db4a6386afdf5ad5ef9 is 50, key is test_row_0/A:col10/1731575629301/Put/seqid=0 2024-11-14T09:13:49,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741846_1022 (size=14491) 2024-11-14T09:13:49,473 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/4f384c57c2474db4a6386afdf5ad5ef9 2024-11-14T09:13:49,496 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/069b66ade5cf4261ac1a23824628912d is 50, key is test_row_0/B:col10/1731575629301/Put/seqid=0 2024-11-14T09:13:49,526 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731575629435, max timestamp=1731575629521Num uniques 859; 2024-11-14T09:13:49,526 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731575629435, max timestamp=1731575629521Num uniques 859; 2024-11-14T09:13:49,527 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731575629435, max timestamp=1731575629521Num uniques 859; 2024-11-14T09:13:49,527 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T09:13:49,527 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T09:13:49,527 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T09:13:49,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741847_1023 (size=12151) 2024-11-14T09:13:49,532 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/069b66ade5cf4261ac1a23824628912d 2024-11-14T09:13:49,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/5e96899ed6334936ad9c2327c711e028 is 50, key is test_row_0/C:col10/1731575629301/Put/seqid=0 2024-11-14T09:13:49,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741848_1024 (size=12151) 2024-11-14T09:13:49,570 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/5e96899ed6334936ad9c2327c711e028 2024-11-14T09:13:49,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/4f384c57c2474db4a6386afdf5ad5ef9 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/4f384c57c2474db4a6386afdf5ad5ef9 2024-11-14T09:13:49,597 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/4f384c57c2474db4a6386afdf5ad5ef9, entries=200, sequenceid=134, filesize=14.2 K 2024-11-14T09:13:49,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/069b66ade5cf4261ac1a23824628912d as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/069b66ade5cf4261ac1a23824628912d 2024-11-14T09:13:49,614 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/069b66ade5cf4261ac1a23824628912d, entries=150, sequenceid=134, filesize=11.9 K 2024-11-14T09:13:49,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/5e96899ed6334936ad9c2327c711e028 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/5e96899ed6334936ad9c2327c711e028 2024-11-14T09:13:49,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:49,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49980 deadline: 1731575689605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:49,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:49,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:49,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:49,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:49,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49996 deadline: 1731575689615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:49,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:49,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:49,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:49,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:49,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575689618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:49,630 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/5e96899ed6334936ad9c2327c711e028, entries=150, sequenceid=134, filesize=11.9 K 2024-11-14T09:13:49,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:49,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:49,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:49,631 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for a92123abacf02fce4f65c3ba94256e83 in 202ms, sequenceid=134, compaction requested=true 2024-11-14T09:13:49,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a92123abacf02fce4f65c3ba94256e83: 2024-11-14T09:13:49,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on a92123abacf02fce4f65c3ba94256e83 2024-11-14T09:13:49,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a92123abacf02fce4f65c3ba94256e83:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:13:49,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:13:49,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731575629589, max timestamp=1731575629644Num uniques 1118; 2024-11-14T09:13:49,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a92123abacf02fce4f65c3ba94256e83:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:13:49,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:13:49,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a92123abacf02fce4f65c3ba94256e83:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:13:49,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=12268, suffix heap size=144240,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=268 2024-11-14T09:13:49,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:13:49,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731575629589, max timestamp=1731575629644Num uniques 1118; 2024-11-14T09:13:49,653 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=12268, suffix heap size=144240,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=268 2024-11-14T09:13:49,653 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a92123abacf02fce4f65c3ba94256e83 3/3 column families, dataSize=71.56 KB heapSize=189.09 KB 2024-11-14T09:13:49,655 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:13:49,655 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:13:49,656 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731575629589, max timestamp=1731575629644Num uniques 1118; 2024-11-14T09:13:49,656 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=12268, suffix heap size=144240,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=268 2024-11-14T09:13:49,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=A 2024-11-14T09:13:49,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:13:49,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=B 2024-11-14T09:13:49,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:13:49,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=C 2024-11-14T09:13:49,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:13:49,663 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48154 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:13:49,665 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): a92123abacf02fce4f65c3ba94256e83/B is initiating minor compaction (all files) 2024-11-14T09:13:49,665 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a92123abacf02fce4f65c3ba94256e83/B in TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:49,666 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57514 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:13:49,666 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/9a5626a483f944d494ac17692769a729, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/e7cacf86dde84ea0ae65d6fb0d70417b, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/3e17093d314248e49c7cc0f738ad3e5f, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/069b66ade5cf4261ac1a23824628912d] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp, totalSize=47.0 K 2024-11-14T09:13:49,666 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): a92123abacf02fce4f65c3ba94256e83/A is initiating minor compaction (all files) 2024-11-14T09:13:49,666 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a92123abacf02fce4f65c3ba94256e83/A in TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:49,666 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/48e3bdfc754c47d9a827e09d90ec10e5, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/c309c9d7673144ec8c135c77844a4b17, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/9ab6d0155d964562a5cd5b4716cc8bff, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/4f384c57c2474db4a6386afdf5ad5ef9] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp, totalSize=56.2 K 2024-11-14T09:13:49,668 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a5626a483f944d494ac17692769a729, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731575627955 2024-11-14T09:13:49,669 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48e3bdfc754c47d9a827e09d90ec10e5, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731575627955 2024-11-14T09:13:49,669 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting e7cacf86dde84ea0ae65d6fb0d70417b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731575628097 2024-11-14T09:13:49,669 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting c309c9d7673144ec8c135c77844a4b17, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731575628086 2024-11-14T09:13:49,670 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e17093d314248e49c7cc0f738ad3e5f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1731575628939 2024-11-14T09:13:49,671 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ab6d0155d964562a5cd5b4716cc8bff, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1731575628939 2024-11-14T09:13:49,671 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 069b66ade5cf4261ac1a23824628912d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731575629301 2024-11-14T09:13:49,672 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f384c57c2474db4a6386afdf5ad5ef9, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731575629281 2024-11-14T09:13:49,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/257325ccf4b24fe7b5cc16126b3f2798 is 50, key is test_row_0/A:col10/1731575629633/Put/seqid=0 2024-11-14T09:13:49,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741849_1025 (size=14541) 2024-11-14T09:13:49,740 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a92123abacf02fce4f65c3ba94256e83#A#compaction#13 average throughput is 0.50 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:13:49,743 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/690bcafabc6548f99789cd73462022ef is 50, key is test_row_0/A:col10/1731575629301/Put/seqid=0 2024-11-14T09:13:49,744 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a92123abacf02fce4f65c3ba94256e83#B#compaction#14 average throughput is 0.34 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:13:49,745 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/cee7ef0618e4446ca07773cc4f4bb2f6 is 50, key is test_row_0/B:col10/1731575629301/Put/seqid=0 2024-11-14T09:13:49,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741851_1027 (size=12289) 2024-11-14T09:13:49,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741850_1026 (size=12289) 2024-11-14T09:13:49,794 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731575629763, max timestamp=1731575629785Num uniques 859; 2024-11-14T09:13:49,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731575629763, max timestamp=1731575629785Num uniques 859; 2024-11-14T09:13:49,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:13:49,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:13:49,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731575629763, max timestamp=1731575629785Num uniques 859; 2024-11-14T09:13:49,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:13:49,803 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/690bcafabc6548f99789cd73462022ef as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/690bcafabc6548f99789cd73462022ef 2024-11-14T09:13:49,833 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a92123abacf02fce4f65c3ba94256e83/A of a92123abacf02fce4f65c3ba94256e83 into 690bcafabc6548f99789cd73462022ef(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:13:49,833 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a92123abacf02fce4f65c3ba94256e83: 2024-11-14T09:13:49,835 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., storeName=a92123abacf02fce4f65c3ba94256e83/A, priority=12, startTime=1731575629633; duration=0sec 2024-11-14T09:13:49,836 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:13:49,836 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a92123abacf02fce4f65c3ba94256e83:A 2024-11-14T09:13:49,836 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:13:49,840 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48154 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:13:49,840 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): a92123abacf02fce4f65c3ba94256e83/C is initiating minor compaction (all files) 2024-11-14T09:13:49,840 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a92123abacf02fce4f65c3ba94256e83/C in TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:49,840 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/ac7dc2dcde354dedb5bf0b92634c7b9a, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/9131cf1925354ac0b1a2c87dfa49ef14, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/fe0d594cf40d467d9b9b939c49abfcf4, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/5e96899ed6334936ad9c2327c711e028] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp, totalSize=47.0 K 2024-11-14T09:13:49,843 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac7dc2dcde354dedb5bf0b92634c7b9a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731575627955 2024-11-14T09:13:49,848 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9131cf1925354ac0b1a2c87dfa49ef14, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731575628097 2024-11-14T09:13:49,849 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe0d594cf40d467d9b9b939c49abfcf4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1731575628939 2024-11-14T09:13:49,851 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e96899ed6334936ad9c2327c711e028, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731575629301 2024-11-14T09:13:49,875 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a92123abacf02fce4f65c3ba94256e83#C#compaction#15 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:13:49,876 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731575629840, max timestamp=1731575629863Num uniques 1018; 2024-11-14T09:13:49,877 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/9c83ad8d4dad4ae0bb34183f73968c7d is 50, key is test_row_0/C:col10/1731575629301/Put/seqid=0 2024-11-14T09:13:49,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-14T09:13:49,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731575629840, max timestamp=1731575629863Num uniques 1018; 2024-11-14T09:13:49,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-14T09:13:49,881 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731575629840, max timestamp=1731575629863Num uniques 1018; 2024-11-14T09:13:49,881 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-14T09:13:49,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741852_1028 (size=12289) 2024-11-14T09:13:49,952 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731575629920, max timestamp=1731575629943Num uniques 1077; 2024-11-14T09:13:49,952 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-14T09:13:49,956 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731575629920, max timestamp=1731575629943Num uniques 1077; 2024-11-14T09:13:49,959 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731575629920, max timestamp=1731575629943Num uniques 1077; 2024-11-14T09:13:49,959 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-14T09:13:49,959 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-14T09:13:50,029 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731575629993, max timestamp=1731575630024Num uniques 1086; 2024-11-14T09:13:50,029 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731575629993, max timestamp=1731575630024Num uniques 1086; 2024-11-14T09:13:50,030 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-14T09:13:50,030 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-14T09:13:50,033 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731575629993, max timestamp=1731575630024Num uniques 1086; 2024-11-14T09:13:50,034 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-14T09:13:50,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T09:13:50,086 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:13:50,086 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:13:50,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-14T09:13:50,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T09:13:50,097 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:13:50,098 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:13:50,098 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:13:50,129 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/257325ccf4b24fe7b5cc16126b3f2798 2024-11-14T09:13:50,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:50,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49966 deadline: 1731575690088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:50,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731575630072, max timestamp=1731575630099Num uniques 1095; 2024-11-14T09:13:50,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-14T09:13:50,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:50,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:50,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:50,142 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731575630072, max timestamp=1731575630094Num uniques 1095; 2024-11-14T09:13:50,142 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-14T09:13:50,142 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731575630072, max timestamp=1731575630099Num uniques 1095; 2024-11-14T09:13:50,142 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-14T09:13:50,156 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/c6a295c0796145909cbb0ed78cfcb54b is 50, key is test_row_0/B:col10/1731575629633/Put/seqid=0 2024-11-14T09:13:50,197 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/cee7ef0618e4446ca07773cc4f4bb2f6 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/cee7ef0618e4446ca07773cc4f4bb2f6 2024-11-14T09:13:50,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741853_1029 (size=12151) 2024-11-14T09:13:50,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T09:13:50,238 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a92123abacf02fce4f65c3ba94256e83/B of a92123abacf02fce4f65c3ba94256e83 into cee7ef0618e4446ca07773cc4f4bb2f6(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:13:50,238 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a92123abacf02fce4f65c3ba94256e83: 2024-11-14T09:13:50,239 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., storeName=a92123abacf02fce4f65c3ba94256e83/B, priority=12, startTime=1731575629652; duration=0sec 2024-11-14T09:13:50,239 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:13:50,239 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a92123abacf02fce4f65c3ba94256e83:B 2024-11-14T09:13:50,254 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T09:13:50,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:50,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:13:50,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:50,254 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:50,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:50,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:50,313 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731575630188, max timestamp=1731575630275Num uniques 1054; 2024-11-14T09:13:50,313 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731575630188, max timestamp=1731575630264Num uniques 1054; 2024-11-14T09:13:50,313 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731575630188, max timestamp=1731575630275Num uniques 1054; 2024-11-14T09:13:50,313 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-14T09:13:50,313 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-14T09:13:50,313 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-14T09:13:50,338 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/9c83ad8d4dad4ae0bb34183f73968c7d as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/9c83ad8d4dad4ae0bb34183f73968c7d 2024-11-14T09:13:50,363 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a92123abacf02fce4f65c3ba94256e83/C of a92123abacf02fce4f65c3ba94256e83 into 9c83ad8d4dad4ae0bb34183f73968c7d(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:13:50,363 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a92123abacf02fce4f65c3ba94256e83: 2024-11-14T09:13:50,363 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., storeName=a92123abacf02fce4f65c3ba94256e83/C, priority=12, startTime=1731575629652; duration=0sec 2024-11-14T09:13:50,363 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:13:50,363 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a92123abacf02fce4f65c3ba94256e83:C 2024-11-14T09:13:50,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:50,378 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731575630350, max timestamp=1731575630370Num uniques 1013; 2024-11-14T09:13:50,378 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-11-14T09:13:50,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49996 deadline: 1731575690364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:50,378 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731575630350, max timestamp=1731575630370Num uniques 1013; 2024-11-14T09:13:50,378 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-11-14T09:13:50,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:50,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:50,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:50,384 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731575630350, max timestamp=1731575630370Num uniques 1013; 2024-11-14T09:13:50,385 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-11-14T09:13:50,410 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T09:13:50,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:50,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:13:50,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:50,411 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:50,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:50,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:50,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T09:13:50,444 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731575630420, max timestamp=1731575630440Num uniques 1022; 2024-11-14T09:13:50,445 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=10162, suffix heap size=134256,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-11-14T09:13:50,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731575630420, max timestamp=1731575630440Num uniques 1022; 2024-11-14T09:13:50,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=10162, suffix heap size=134256,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-11-14T09:13:50,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731575630420, max timestamp=1731575630440Num uniques 1022; 2024-11-14T09:13:50,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=10162, suffix heap size=134256,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-11-14T09:13:50,502 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731575630486, max timestamp=1731575630496Num uniques 1081; 2024-11-14T09:13:50,502 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=8286, suffix heap size=140392,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=181 2024-11-14T09:13:50,502 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731575630486, max timestamp=1731575630496Num uniques 1081; 2024-11-14T09:13:50,502 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731575630486, max timestamp=1731575630496Num uniques 1081; 2024-11-14T09:13:50,502 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=8286, suffix heap size=140392,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=181 2024-11-14T09:13:50,502 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=8286, suffix heap size=140392,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=181 2024-11-14T09:13:50,547 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=340, cellSize=15570, totalHeapSize=35600, min timestamp=1731575630515, max timestamp=1731575630542Num uniques 1040; 2024-11-14T09:13:50,547 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=340, cellSize=15570, totalHeapSize=35600, min timestamp=1731575630515, max timestamp=1731575630542Num uniques 1040; 2024-11-14T09:13:50,547 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=15570, suffix heap size=136128,new segment heap size=35600 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=340 2024-11-14T09:13:50,547 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=15570, suffix heap size=136128,new segment heap size=35600 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=340 2024-11-14T09:13:50,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1731575630515, max timestamp=1731575630542Num uniques 1040; 2024-11-14T09:13:50,551 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=10990, suffix heap size=136128,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=240 2024-11-14T09:13:50,566 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T09:13:50,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:50,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:13:50,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:50,567 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:50,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:50,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:50,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:50,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49996 deadline: 1731575690581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:50,598 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=349, cellSize=15984, totalHeapSize=36536, min timestamp=1731575630570, max timestamp=1731575630589Num uniques 1199; 2024-11-14T09:13:50,599 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=54914, new segment data size=15984, suffix heap size=152664,new segment heap size=36536 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1199, new segment cells count=349 2024-11-14T09:13:50,599 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=349, cellSize=15984, totalHeapSize=36536, min timestamp=1731575630570, max timestamp=1731575630589Num uniques 1199; 2024-11-14T09:13:50,599 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=54914, new segment data size=15984, suffix heap size=152664,new segment heap size=36536 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1199, new segment cells count=349 2024-11-14T09:13:50,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:50,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:50,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:50,602 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=349, cellSize=15984, totalHeapSize=36536, min timestamp=1731575630570, max timestamp=1731575630589Num uniques 1099; 2024-11-14T09:13:50,603 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=15984, suffix heap size=142264,new segment heap size=36536 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=349 2024-11-14T09:13:50,604 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/c6a295c0796145909cbb0ed78cfcb54b 2024-11-14T09:13:50,629 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/e21e157f4b5040d9bfc7d8e68adbffa7 is 50, key is test_row_0/C:col10/1731575629633/Put/seqid=0 2024-11-14T09:13:50,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741854_1030 (size=12151) 2024-11-14T09:13:50,640 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/e21e157f4b5040d9bfc7d8e68adbffa7 2024-11-14T09:13:50,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:50,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575690641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:50,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:50,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:50,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:50,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:50,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:50,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49966 deadline: 1731575690639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:50,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49980 deadline: 1731575690643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:50,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/257325ccf4b24fe7b5cc16126b3f2798 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/257325ccf4b24fe7b5cc16126b3f2798 2024-11-14T09:13:50,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:50,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:50,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:50,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:50,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:50,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:50,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:50,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575690648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:50,657 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:50,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:50,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:50,664 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/257325ccf4b24fe7b5cc16126b3f2798, entries=200, sequenceid=172, filesize=14.2 K 2024-11-14T09:13:50,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/c6a295c0796145909cbb0ed78cfcb54b as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/c6a295c0796145909cbb0ed78cfcb54b 2024-11-14T09:13:50,683 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/c6a295c0796145909cbb0ed78cfcb54b, entries=150, sequenceid=172, filesize=11.9 K 2024-11-14T09:13:50,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/e21e157f4b5040d9bfc7d8e68adbffa7 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/e21e157f4b5040d9bfc7d8e68adbffa7 2024-11-14T09:13:50,700 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/e21e157f4b5040d9bfc7d8e68adbffa7, entries=150, sequenceid=172, filesize=11.9 K 2024-11-14T09:13:50,702 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~95.81 KB/98112, currentSize=154.31 KB/158010 for a92123abacf02fce4f65c3ba94256e83 in 1050ms, sequenceid=172, compaction requested=false 2024-11-14T09:13:50,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a92123abacf02fce4f65c3ba94256e83: 2024-11-14T09:13:50,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on a92123abacf02fce4f65c3ba94256e83 2024-11-14T09:13:50,709 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a92123abacf02fce4f65c3ba94256e83 3/3 column families, dataSize=161.02 KB heapSize=446.86 KB 2024-11-14T09:13:50,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=A 2024-11-14T09:13:50,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:13:50,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=B 2024-11-14T09:13:50,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:13:50,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=C 2024-11-14T09:13:50,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:13:50,718 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T09:13:50,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T09:13:50,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T09:13:50,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:50,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:13:50,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:50,728 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:50,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:50,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:50,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/0da76ca844b54096b8bd1786dd0a979e is 50, key is test_row_0/A:col10/1731575630630/Put/seqid=0 2024-11-14T09:13:50,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:50,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49996 deadline: 1731575690751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:50,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:50,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:50,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:50,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:50,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575690755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:50,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:50,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:50,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:50,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:50,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49966 deadline: 1731575690756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:50,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:50,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:50,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:50,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:50,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:50,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49980 deadline: 1731575690765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:50,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575690766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:50,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:50,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:50,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:50,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:50,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:50,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:50,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741855_1031 (size=14741) 2024-11-14T09:13:50,817 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-14T09:13:50,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:50,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49996 deadline: 1731575690865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:50,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:50,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:50,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:50,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T09:13:50,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:50,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:13:50,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:50,886 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:50,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:50,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:50,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:50,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575690975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:50,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:50,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:50,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:50,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:50,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49980 deadline: 1731575690986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:50,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:50,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:50,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:50,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:50,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49966 deadline: 1731575690986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:50,992 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:50,992 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:50,992 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:50,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:50,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575690986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:50,997 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:50,997 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:50,997 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:51,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T09:13:51,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:51,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:13:51,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:51,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:51,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:51,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:51,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:51,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49996 deadline: 1731575691087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:51,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:51,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:51,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:51,183 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/0da76ca844b54096b8bd1786dd0a979e 2024-11-14T09:13:51,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T09:13:51,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:51,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:13:51,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:51,202 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:51,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:51,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:51,213 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/f72ffa4260b643c292836c07077703ec is 50, key is test_row_0/B:col10/1731575630630/Put/seqid=0 2024-11-14T09:13:51,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T09:13:51,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741856_1032 (size=12301) 2024-11-14T09:13:51,261 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/f72ffa4260b643c292836c07077703ec 2024-11-14T09:13:51,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/515ce2d699ac46fdbe1e9dcc6d4c1918 is 50, key is test_row_0/C:col10/1731575630630/Put/seqid=0 2024-11-14T09:13:51,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:51,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575691290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:51,293 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:51,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:51,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:51,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:51,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49980 deadline: 1731575691305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:51,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:51,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:51,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:51,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:51,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:51,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49966 deadline: 1731575691305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:51,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575691306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:51,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:51,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:51,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:51,316 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:51,316 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:51,316 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:51,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741857_1033 (size=12301) 2024-11-14T09:13:51,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T09:13:51,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:51,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:13:51,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:51,357 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:51,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:51,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:51,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:51,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49996 deadline: 1731575691395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:51,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:51,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:51,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:51,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T09:13:51,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:51,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:13:51,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:51,514 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:51,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:51,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:51,674 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T09:13:51,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:51,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:13:51,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:51,675 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:51,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:51,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:51,744 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/515ce2d699ac46fdbe1e9dcc6d4c1918 2024-11-14T09:13:51,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/0da76ca844b54096b8bd1786dd0a979e as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/0da76ca844b54096b8bd1786dd0a979e 2024-11-14T09:13:51,794 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/0da76ca844b54096b8bd1786dd0a979e, entries=200, sequenceid=384, filesize=14.4 K 2024-11-14T09:13:51,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/f72ffa4260b643c292836c07077703ec as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/f72ffa4260b643c292836c07077703ec 2024-11-14T09:13:51,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:51,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575691805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:51,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:51,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:51,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:51,825 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/f72ffa4260b643c292836c07077703ec, entries=150, sequenceid=384, filesize=12.0 K 2024-11-14T09:13:51,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:51,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49980 deadline: 1731575691826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:51,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:51,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:51,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:51,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:51,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49966 deadline: 1731575691827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:51,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:51,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:51,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:51,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:51,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575691827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:51,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/515ce2d699ac46fdbe1e9dcc6d4c1918 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/515ce2d699ac46fdbe1e9dcc6d4c1918 2024-11-14T09:13:51,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T09:13:51,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:51,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:51,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:51,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:51,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:13:51,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:51,834 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:51,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:51,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:51,849 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/515ce2d699ac46fdbe1e9dcc6d4c1918, entries=150, sequenceid=384, filesize=12.0 K 2024-11-14T09:13:51,860 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~446.81 KB/457536, currentSize=26.84 KB/27480 for a92123abacf02fce4f65c3ba94256e83 in 1151ms, sequenceid=384, compaction requested=true 2024-11-14T09:13:51,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a92123abacf02fce4f65c3ba94256e83: 2024-11-14T09:13:51,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a92123abacf02fce4f65c3ba94256e83:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:13:51,861 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:13:51,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:13:51,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a92123abacf02fce4f65c3ba94256e83:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:13:51,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:13:51,861 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:13:51,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a92123abacf02fce4f65c3ba94256e83:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:13:51,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:13:51,863 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41571 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:13:51,863 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): a92123abacf02fce4f65c3ba94256e83/A is initiating minor compaction (all files) 2024-11-14T09:13:51,863 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a92123abacf02fce4f65c3ba94256e83/A in TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:51,864 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/690bcafabc6548f99789cd73462022ef, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/257325ccf4b24fe7b5cc16126b3f2798, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/0da76ca844b54096b8bd1786dd0a979e] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp, totalSize=40.6 K 2024-11-14T09:13:51,864 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36741 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:13:51,864 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): a92123abacf02fce4f65c3ba94256e83/B is initiating minor compaction (all files) 2024-11-14T09:13:51,864 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a92123abacf02fce4f65c3ba94256e83/B in TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:51,865 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/cee7ef0618e4446ca07773cc4f4bb2f6, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/c6a295c0796145909cbb0ed78cfcb54b, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/f72ffa4260b643c292836c07077703ec] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp, totalSize=35.9 K 2024-11-14T09:13:51,865 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 690bcafabc6548f99789cd73462022ef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731575629301 2024-11-14T09:13:51,865 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting cee7ef0618e4446ca07773cc4f4bb2f6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731575629301 2024-11-14T09:13:51,866 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 257325ccf4b24fe7b5cc16126b3f2798, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731575629600 2024-11-14T09:13:51,866 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting c6a295c0796145909cbb0ed78cfcb54b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731575629615 2024-11-14T09:13:51,867 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0da76ca844b54096b8bd1786dd0a979e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1731575630630 2024-11-14T09:13:51,867 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting f72ffa4260b643c292836c07077703ec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1731575630630 2024-11-14T09:13:51,894 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a92123abacf02fce4f65c3ba94256e83#A#compaction#21 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:13:51,895 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/02500cc37f6041a8ab34a44f4f8b7081 is 50, key is test_row_0/A:col10/1731575630630/Put/seqid=0 2024-11-14T09:13:51,901 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a92123abacf02fce4f65c3ba94256e83#B#compaction#22 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:13:51,902 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/7c5e02155102424e9cc8eb1e4ab6e209 is 50, key is test_row_0/B:col10/1731575630630/Put/seqid=0 2024-11-14T09:13:51,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741858_1034 (size=12541) 2024-11-14T09:13:51,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741859_1035 (size=12541) 2024-11-14T09:13:51,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on a92123abacf02fce4f65c3ba94256e83 2024-11-14T09:13:51,963 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a92123abacf02fce4f65c3ba94256e83 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T09:13:51,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=A 2024-11-14T09:13:51,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:13:51,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=B 2024-11-14T09:13:51,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:13:51,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=C 2024-11-14T09:13:51,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:13:51,982 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/7c5e02155102424e9cc8eb1e4ab6e209 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/7c5e02155102424e9cc8eb1e4ab6e209 2024-11-14T09:13:51,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/4ca60702129b4adaa7de671c1e7f13e8 is 50, key is test_row_0/A:col10/1731575630729/Put/seqid=0 2024-11-14T09:13:51,988 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/02500cc37f6041a8ab34a44f4f8b7081 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/02500cc37f6041a8ab34a44f4f8b7081 2024-11-14T09:13:51,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T09:13:51,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:51,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:13:51,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:51,990 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:51,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:51,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:52,003 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a92123abacf02fce4f65c3ba94256e83/B of a92123abacf02fce4f65c3ba94256e83 into 7c5e02155102424e9cc8eb1e4ab6e209(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:13:52,004 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a92123abacf02fce4f65c3ba94256e83: 2024-11-14T09:13:52,004 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., storeName=a92123abacf02fce4f65c3ba94256e83/B, priority=13, startTime=1731575631861; duration=0sec 2024-11-14T09:13:52,004 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:13:52,004 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a92123abacf02fce4f65c3ba94256e83:B 2024-11-14T09:13:52,004 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:13:52,008 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36741 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:13:52,008 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): a92123abacf02fce4f65c3ba94256e83/C is initiating minor compaction (all files) 2024-11-14T09:13:52,008 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a92123abacf02fce4f65c3ba94256e83/C in TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:52,008 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/9c83ad8d4dad4ae0bb34183f73968c7d, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/e21e157f4b5040d9bfc7d8e68adbffa7, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/515ce2d699ac46fdbe1e9dcc6d4c1918] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp, totalSize=35.9 K 2024-11-14T09:13:52,009 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a92123abacf02fce4f65c3ba94256e83/A of a92123abacf02fce4f65c3ba94256e83 into 02500cc37f6041a8ab34a44f4f8b7081(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:13:52,009 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a92123abacf02fce4f65c3ba94256e83: 2024-11-14T09:13:52,009 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., storeName=a92123abacf02fce4f65c3ba94256e83/A, priority=13, startTime=1731575631860; duration=0sec 2024-11-14T09:13:52,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741860_1036 (size=14741) 2024-11-14T09:13:52,010 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c83ad8d4dad4ae0bb34183f73968c7d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731575629301 2024-11-14T09:13:52,011 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting e21e157f4b5040d9bfc7d8e68adbffa7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731575629615 2024-11-14T09:13:52,012 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 515ce2d699ac46fdbe1e9dcc6d4c1918, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1731575630630 2024-11-14T09:13:52,014 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:13:52,014 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a92123abacf02fce4f65c3ba94256e83:A 2024-11-14T09:13:52,040 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a92123abacf02fce4f65c3ba94256e83#C#compaction#24 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:13:52,041 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/b5b75880497b4eaaaf77b253d910ef77 is 50, key is test_row_0/C:col10/1731575630630/Put/seqid=0 2024-11-14T09:13:52,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741861_1037 (size=12541) 2024-11-14T09:13:52,115 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-14T09:13:52,116 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-14T09:13:52,118 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T09:13:52,118 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-14T09:13:52,118 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-14T09:13:52,118 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-14T09:13:52,119 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-14T09:13:52,119 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-14T09:13:52,120 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/b5b75880497b4eaaaf77b253d910ef77 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/b5b75880497b4eaaaf77b253d910ef77 2024-11-14T09:13:52,139 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a92123abacf02fce4f65c3ba94256e83/C of a92123abacf02fce4f65c3ba94256e83 into b5b75880497b4eaaaf77b253d910ef77(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:13:52,140 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a92123abacf02fce4f65c3ba94256e83: 2024-11-14T09:13:52,140 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., storeName=a92123abacf02fce4f65c3ba94256e83/C, priority=13, startTime=1731575631861; duration=0sec 2024-11-14T09:13:52,140 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:13:52,140 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a92123abacf02fce4f65c3ba94256e83:C 2024-11-14T09:13:52,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T09:13:52,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:52,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:13:52,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:52,144 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:52,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:52,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:52,185 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731575632094, max timestamp=1731575632180Num uniques 859; 2024-11-14T09:13:52,186 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:13:52,188 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731575632094, max timestamp=1731575632180Num uniques 859; 2024-11-14T09:13:52,188 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:13:52,191 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731575632094, max timestamp=1731575632180Num uniques 859; 2024-11-14T09:13:52,191 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:13:52,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T09:13:52,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T09:13:52,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:52,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:13:52,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:52,303 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:52,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:52,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:52,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:52,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49996 deadline: 1731575692316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:52,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:52,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:52,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:52,412 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/4ca60702129b4adaa7de671c1e7f13e8 2024-11-14T09:13:52,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:52,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49996 deadline: 1731575692425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:52,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:52,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:52,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:52,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/f0499f50d9494bf199c90136c45743d9 is 50, key is test_row_0/B:col10/1731575630729/Put/seqid=0 2024-11-14T09:13:52,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741862_1038 (size=12301) 2024-11-14T09:13:52,443 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/f0499f50d9494bf199c90136c45743d9 2024-11-14T09:13:52,457 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/60b218fdadf9459b81501264fc7f517d is 50, key is test_row_0/C:col10/1731575630729/Put/seqid=0 2024-11-14T09:13:52,458 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T09:13:52,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:52,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:13:52,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:52,458 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:52,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:52,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:52,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741863_1039 (size=12301) 2024-11-14T09:13:52,479 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/60b218fdadf9459b81501264fc7f517d 2024-11-14T09:13:52,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/4ca60702129b4adaa7de671c1e7f13e8 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/4ca60702129b4adaa7de671c1e7f13e8 2024-11-14T09:13:52,503 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/4ca60702129b4adaa7de671c1e7f13e8, entries=200, sequenceid=394, filesize=14.4 K 2024-11-14T09:13:52,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/f0499f50d9494bf199c90136c45743d9 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/f0499f50d9494bf199c90136c45743d9 2024-11-14T09:13:52,517 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/f0499f50d9494bf199c90136c45743d9, entries=150, sequenceid=394, filesize=12.0 K 2024-11-14T09:13:52,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/60b218fdadf9459b81501264fc7f517d as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/60b218fdadf9459b81501264fc7f517d 2024-11-14T09:13:52,532 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/60b218fdadf9459b81501264fc7f517d, entries=150, sequenceid=394, filesize=12.0 K 2024-11-14T09:13:52,533 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for a92123abacf02fce4f65c3ba94256e83 in 570ms, sequenceid=394, compaction requested=false 2024-11-14T09:13:52,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a92123abacf02fce4f65c3ba94256e83: 2024-11-14T09:13:52,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T09:13:52,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:52,614 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing a92123abacf02fce4f65c3ba94256e83 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-14T09:13:52,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=A 2024-11-14T09:13:52,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:13:52,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=B 2024-11-14T09:13:52,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:13:52,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=C 2024-11-14T09:13:52,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:13:52,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/98c612af874e4e73b80d84397de05bb2 is 50, key is test_row_0/A:col10/1731575632314/Put/seqid=0 2024-11-14T09:13:52,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on a92123abacf02fce4f65c3ba94256e83 2024-11-14T09:13:52,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:13:52,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741864_1040 (size=12301) 2024-11-14T09:13:52,658 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/98c612af874e4e73b80d84397de05bb2 2024-11-14T09:13:52,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/376f2e79eb3c4908b44d2a11fe964418 is 50, key is test_row_0/B:col10/1731575632314/Put/seqid=0 2024-11-14T09:13:52,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741865_1041 (size=12301) 2024-11-14T09:13:52,693 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/376f2e79eb3c4908b44d2a11fe964418 2024-11-14T09:13:52,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:52,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49996 deadline: 1731575692698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:52,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:52,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:52,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:52,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/2a0afeec1f044ef3aa64c408e0b8a1a5 is 50, key is test_row_0/C:col10/1731575632314/Put/seqid=0 2024-11-14T09:13:52,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741866_1042 (size=12301) 2024-11-14T09:13:52,730 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/2a0afeec1f044ef3aa64c408e0b8a1a5 2024-11-14T09:13:52,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/98c612af874e4e73b80d84397de05bb2 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/98c612af874e4e73b80d84397de05bb2 2024-11-14T09:13:52,753 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/98c612af874e4e73b80d84397de05bb2, entries=150, sequenceid=434, filesize=12.0 K 2024-11-14T09:13:52,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/376f2e79eb3c4908b44d2a11fe964418 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/376f2e79eb3c4908b44d2a11fe964418 2024-11-14T09:13:52,764 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/376f2e79eb3c4908b44d2a11fe964418, entries=150, sequenceid=434, filesize=12.0 K 2024-11-14T09:13:52,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/2a0afeec1f044ef3aa64c408e0b8a1a5 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/2a0afeec1f044ef3aa64c408e0b8a1a5 2024-11-14T09:13:52,773 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/2a0afeec1f044ef3aa64c408e0b8a1a5, entries=150, sequenceid=434, filesize=12.0 K 2024-11-14T09:13:52,776 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for a92123abacf02fce4f65c3ba94256e83 in 162ms, sequenceid=434, compaction requested=true 2024-11-14T09:13:52,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for a92123abacf02fce4f65c3ba94256e83: 2024-11-14T09:13:52,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:52,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-14T09:13:52,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-14T09:13:52,784 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-14T09:13:52,784 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6830 sec 2024-11-14T09:13:52,789 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 2.7000 sec 2024-11-14T09:13:52,819 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a92123abacf02fce4f65c3ba94256e83 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-14T09:13:52,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=A 2024-11-14T09:13:52,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on a92123abacf02fce4f65c3ba94256e83 2024-11-14T09:13:52,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:13:52,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=B 2024-11-14T09:13:52,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:13:52,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=C 2024-11-14T09:13:52,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:13:52,829 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/40af6745cad845d2a168d961a77100f7 is 50, key is test_row_0/A:col10/1731575632639/Put/seqid=0 2024-11-14T09:13:52,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741867_1043 (size=14741) 2024-11-14T09:13:52,848 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/40af6745cad845d2a168d961a77100f7 2024-11-14T09:13:52,868 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/0c8bbd0bde3d45b398f632cff954645d is 50, key is test_row_0/B:col10/1731575632639/Put/seqid=0 2024-11-14T09:13:52,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731575632865, max timestamp=1731575632873Num uniques 859; 2024-11-14T09:13:52,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731575632865, max timestamp=1731575632873Num uniques 859; 2024-11-14T09:13:52,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731575632865, max timestamp=1731575632873Num uniques 859; 2024-11-14T09:13:52,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:52,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:13:52,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:13:52,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:13:52,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49980 deadline: 1731575692866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:52,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:52,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:52,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:52,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741868_1044 (size=12301) 2024-11-14T09:13:52,913 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/0c8bbd0bde3d45b398f632cff954645d 2024-11-14T09:13:52,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:52,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49996 deadline: 1731575692915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:52,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:52,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49966 deadline: 1731575692924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:52,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:52,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:52,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:52,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:52,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:52,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:52,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:52,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575692926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:52,932 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:52,932 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:52,932 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:52,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:52,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575692928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:52,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:52,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:52,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:52,936 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/d91dd276de3a498caf953e63ece2dc5d is 50, key is test_row_0/C:col10/1731575632639/Put/seqid=0 2024-11-14T09:13:52,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741869_1045 (size=12301) 2024-11-14T09:13:52,980 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/d91dd276de3a498caf953e63ece2dc5d 2024-11-14T09:13:52,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:52,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49980 deadline: 1731575692985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:52,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:52,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:52,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:52,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/40af6745cad845d2a168d961a77100f7 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/40af6745cad845d2a168d961a77100f7 2024-11-14T09:13:53,010 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/40af6745cad845d2a168d961a77100f7, entries=200, sequenceid=445, filesize=14.4 K 2024-11-14T09:13:53,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/0c8bbd0bde3d45b398f632cff954645d as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/0c8bbd0bde3d45b398f632cff954645d 2024-11-14T09:13:53,023 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/0c8bbd0bde3d45b398f632cff954645d, entries=150, sequenceid=445, filesize=12.0 K 2024-11-14T09:13:53,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/d91dd276de3a498caf953e63ece2dc5d as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/d91dd276de3a498caf953e63ece2dc5d 2024-11-14T09:13:53,033 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/d91dd276de3a498caf953e63ece2dc5d, entries=150, sequenceid=445, filesize=12.0 K 2024-11-14T09:13:53,036 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for a92123abacf02fce4f65c3ba94256e83 in 216ms, sequenceid=445, compaction requested=true 2024-11-14T09:13:53,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a92123abacf02fce4f65c3ba94256e83: 2024-11-14T09:13:53,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a92123abacf02fce4f65c3ba94256e83:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:13:53,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:13:53,036 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:13:53,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a92123abacf02fce4f65c3ba94256e83:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:13:53,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:13:53,036 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:13:53,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a92123abacf02fce4f65c3ba94256e83:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:13:53,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:13:53,041 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49444 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:13:53,041 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): a92123abacf02fce4f65c3ba94256e83/B is initiating minor compaction (all files) 2024-11-14T09:13:53,041 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a92123abacf02fce4f65c3ba94256e83/B in TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:53,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on a92123abacf02fce4f65c3ba94256e83 2024-11-14T09:13:53,042 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/7c5e02155102424e9cc8eb1e4ab6e209, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/f0499f50d9494bf199c90136c45743d9, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/376f2e79eb3c4908b44d2a11fe964418, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/0c8bbd0bde3d45b398f632cff954645d] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp, totalSize=48.3 K 2024-11-14T09:13:53,042 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54324 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:13:53,042 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): a92123abacf02fce4f65c3ba94256e83/A is initiating minor compaction (all files) 2024-11-14T09:13:53,042 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a92123abacf02fce4f65c3ba94256e83/A in TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:53,043 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/02500cc37f6041a8ab34a44f4f8b7081, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/4ca60702129b4adaa7de671c1e7f13e8, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/98c612af874e4e73b80d84397de05bb2, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/40af6745cad845d2a168d961a77100f7] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp, totalSize=53.1 K 2024-11-14T09:13:53,043 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c5e02155102424e9cc8eb1e4ab6e209, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1731575630630 2024-11-14T09:13:53,044 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 02500cc37f6041a8ab34a44f4f8b7081, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1731575630630 2024-11-14T09:13:53,044 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0499f50d9494bf199c90136c45743d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731575630729 2024-11-14T09:13:53,045 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ca60702129b4adaa7de671c1e7f13e8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731575630721 2024-11-14T09:13:53,045 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 376f2e79eb3c4908b44d2a11fe964418, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1731575632297 2024-11-14T09:13:53,046 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a92123abacf02fce4f65c3ba94256e83 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-14T09:13:53,047 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731575632904, max timestamp=1731575633042Num uniques 1018; 2024-11-14T09:13:53,047 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-14T09:13:53,047 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731575632904, max timestamp=1731575633042Num uniques 1018; 2024-11-14T09:13:53,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-14T09:13:53,048 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 98c612af874e4e73b80d84397de05bb2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1731575632297 2024-11-14T09:13:53,049 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 40af6745cad845d2a168d961a77100f7, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1731575632639 2024-11-14T09:13:53,049 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c8bbd0bde3d45b398f632cff954645d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1731575632639 2024-11-14T09:13:53,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=A 2024-11-14T09:13:53,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:13:53,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=B 2024-11-14T09:13:53,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:13:53,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=C 2024-11-14T09:13:53,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-14T09:13:53,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/41f9beb335a24305902272a41cb276d1 is 50, key is test_row_0/A:col10/1731575633039/Put/seqid=0 2024-11-14T09:13:53,087 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a92123abacf02fce4f65c3ba94256e83#B#compaction#34 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:13:53,088 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/a1ca9c7f90b6432f836343497117e2ca is 50, key is test_row_0/B:col10/1731575632639/Put/seqid=0 2024-11-14T09:13:53,099 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a92123abacf02fce4f65c3ba94256e83#A#compaction#35 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:13:53,100 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/1ef08393e3284940ab6e0c74c5403042 is 50, key is test_row_0/A:col10/1731575632639/Put/seqid=0 2024-11-14T09:13:53,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:53,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:53,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49996 deadline: 1731575693086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:53,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575693086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:53,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:53,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:53,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:53,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:53,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:53,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:53,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:53,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49966 deadline: 1731575693102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:53,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:53,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:53,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:53,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:53,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575693101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:53,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:53,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:53,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:53,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741870_1046 (size=17181) 2024-11-14T09:13:53,134 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/41f9beb335a24305902272a41cb276d1 2024-11-14T09:13:53,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741871_1047 (size=12677) 2024-11-14T09:13:53,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741872_1048 (size=12677) 2024-11-14T09:13:53,174 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/6770f19af96c42de8198cef9aaa3af4c is 50, key is test_row_0/B:col10/1731575633039/Put/seqid=0 2024-11-14T09:13:53,183 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/1ef08393e3284940ab6e0c74c5403042 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/1ef08393e3284940ab6e0c74c5403042 2024-11-14T09:13:53,191 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a92123abacf02fce4f65c3ba94256e83/A of a92123abacf02fce4f65c3ba94256e83 into 1ef08393e3284940ab6e0c74c5403042(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:13:53,192 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a92123abacf02fce4f65c3ba94256e83: 2024-11-14T09:13:53,192 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., storeName=a92123abacf02fce4f65c3ba94256e83/A, priority=12, startTime=1731575633036; duration=0sec 2024-11-14T09:13:53,192 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:13:53,192 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a92123abacf02fce4f65c3ba94256e83:A 2024-11-14T09:13:53,192 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:13:53,197 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49444 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:13:53,197 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): a92123abacf02fce4f65c3ba94256e83/C is initiating minor compaction (all files) 2024-11-14T09:13:53,197 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a92123abacf02fce4f65c3ba94256e83/C in TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:53,198 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/b5b75880497b4eaaaf77b253d910ef77, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/60b218fdadf9459b81501264fc7f517d, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/2a0afeec1f044ef3aa64c408e0b8a1a5, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/d91dd276de3a498caf953e63ece2dc5d] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp, totalSize=48.3 K 2024-11-14T09:13:53,198 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting b5b75880497b4eaaaf77b253d910ef77, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1731575630630 2024-11-14T09:13:53,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:53,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49980 deadline: 1731575693195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:53,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:53,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:53,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:53,202 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 60b218fdadf9459b81501264fc7f517d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731575630729 2024-11-14T09:13:53,203 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a0afeec1f044ef3aa64c408e0b8a1a5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1731575632297 2024-11-14T09:13:53,204 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting d91dd276de3a498caf953e63ece2dc5d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1731575632639 2024-11-14T09:13:53,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741873_1049 (size=12301) 2024-11-14T09:13:53,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:53,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575693205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:53,216 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/6770f19af96c42de8198cef9aaa3af4c 2024-11-14T09:13:53,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:53,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49996 deadline: 1731575693206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:53,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:53,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:53,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:53,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:53,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:53,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:53,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:53,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49966 deadline: 1731575693216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:53,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:53,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:53,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:53,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:53,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575693216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:53,224 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:53,224 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:53,224 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:53,227 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a92123abacf02fce4f65c3ba94256e83#C#compaction#37 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:13:53,228 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/19f0b5abff5344239b2d88b38d790861 is 50, key is test_row_0/C:col10/1731575632639/Put/seqid=0 2024-11-14T09:13:53,230 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/5fcc5458fae24cb3884544764d1e8421 is 50, key is test_row_0/C:col10/1731575633039/Put/seqid=0 2024-11-14T09:13:53,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741875_1051 (size=12301) 2024-11-14T09:13:53,286 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/5fcc5458fae24cb3884544764d1e8421 2024-11-14T09:13:53,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741874_1050 (size=12677) 2024-11-14T09:13:53,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/41f9beb335a24305902272a41cb276d1 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/41f9beb335a24305902272a41cb276d1 2024-11-14T09:13:53,300 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/19f0b5abff5344239b2d88b38d790861 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/19f0b5abff5344239b2d88b38d790861 2024-11-14T09:13:53,304 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/41f9beb335a24305902272a41cb276d1, entries=250, sequenceid=483, filesize=16.8 K 2024-11-14T09:13:53,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/6770f19af96c42de8198cef9aaa3af4c as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/6770f19af96c42de8198cef9aaa3af4c 2024-11-14T09:13:53,311 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a92123abacf02fce4f65c3ba94256e83/C of a92123abacf02fce4f65c3ba94256e83 into 19f0b5abff5344239b2d88b38d790861(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:13:53,311 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a92123abacf02fce4f65c3ba94256e83: 2024-11-14T09:13:53,311 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., storeName=a92123abacf02fce4f65c3ba94256e83/C, priority=12, startTime=1731575633036; duration=0sec 2024-11-14T09:13:53,312 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:13:53,312 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a92123abacf02fce4f65c3ba94256e83:C 2024-11-14T09:13:53,316 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/6770f19af96c42de8198cef9aaa3af4c, entries=150, sequenceid=483, filesize=12.0 K 2024-11-14T09:13:53,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/5fcc5458fae24cb3884544764d1e8421 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/5fcc5458fae24cb3884544764d1e8421 2024-11-14T09:13:53,327 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/5fcc5458fae24cb3884544764d1e8421, entries=150, sequenceid=483, filesize=12.0 K 2024-11-14T09:13:53,328 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~69.33 KB/70990, heapSize ~188.91 KB/193440, currentSize=114.05 KB/116790 for a92123abacf02fce4f65c3ba94256e83 in 285ms, sequenceid=483, compaction requested=false 2024-11-14T09:13:53,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a92123abacf02fce4f65c3ba94256e83: 2024-11-14T09:13:53,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on a92123abacf02fce4f65c3ba94256e83 2024-11-14T09:13:53,437 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731575633080, max timestamp=1731575633434Num uniques 859; 2024-11-14T09:13:53,438 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731575633080, max timestamp=1731575633434Num uniques 859; 2024-11-14T09:13:53,438 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:13:53,438 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:13:53,438 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a92123abacf02fce4f65c3ba94256e83 3/3 column families, dataSize=64.85 KB heapSize=184.09 KB 2024-11-14T09:13:53,440 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731575633080, max timestamp=1731575633434Num uniques 859; 2024-11-14T09:13:53,440 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-14T09:13:53,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=A 2024-11-14T09:13:53,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:13:53,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=B 2024-11-14T09:13:53,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:13:53,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=C 2024-11-14T09:13:53,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:13:53,453 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/93d10d3b98154fdfa4e8e43a0d8fb0d5 is 50, key is test_row_0/A:col10/1731575633436/Put/seqid=0 2024-11-14T09:13:53,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741876_1052 (size=17181) 2024-11-14T09:13:53,498 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731575633467, max timestamp=1731575633492Num uniques 859; 2024-11-14T09:13:53,498 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T09:13:53,503 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731575633467, max timestamp=1731575633492Num uniques 859; 2024-11-14T09:13:53,503 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T09:13:53,504 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731575633467, max timestamp=1731575633492Num uniques 859; 2024-11-14T09:13:53,504 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T09:13:53,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:53,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:53,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575693541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:53,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731575633534, max timestamp=1731575633553Num uniques 1118; 2024-11-14T09:13:53,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731575633534, max timestamp=1731575633553Num uniques 1118; 2024-11-14T09:13:53,559 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=14558, suffix heap size=144240,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=318 2024-11-14T09:13:53,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575693539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:53,559 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731575633534, max timestamp=1731575633553Num uniques 1118; 2024-11-14T09:13:53,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=14558, suffix heap size=144240,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=318 2024-11-14T09:13:53,560 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=14558, suffix heap size=144240,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=318 2024-11-14T09:13:53,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:53,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:53,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:53,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:53,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:53,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:53,572 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/a1ca9c7f90b6432f836343497117e2ca as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/a1ca9c7f90b6432f836343497117e2ca 2024-11-14T09:13:53,586 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a92123abacf02fce4f65c3ba94256e83/B of a92123abacf02fce4f65c3ba94256e83 into a1ca9c7f90b6432f836343497117e2ca(size=12.4 K), total size for store is 24.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:13:53,586 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a92123abacf02fce4f65c3ba94256e83: 2024-11-14T09:13:53,586 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., storeName=a92123abacf02fce4f65c3ba94256e83/B, priority=12, startTime=1731575633036; duration=0sec 2024-11-14T09:13:53,586 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:13:53,587 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a92123abacf02fce4f65c3ba94256e83:B 2024-11-14T09:13:53,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:53,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49966 deadline: 1731575693620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:53,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:53,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:53,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:53,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:53,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49980 deadline: 1731575693622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:53,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:53,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:53,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:53,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:53,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49996 deadline: 1731575693624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:53,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:53,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:53,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:53,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:53,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575693665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:53,669 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:53,669 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:53,669 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:53,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:53,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575693665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:53,671 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:53,671 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:53,671 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:53,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:53,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49966 deadline: 1731575693735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:53,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:53,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:53,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:53,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:53,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49980 deadline: 1731575693735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:53,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:53,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:53,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:53,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:53,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49996 deadline: 1731575693736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:53,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:53,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:53,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:53,863 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=508 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/93d10d3b98154fdfa4e8e43a0d8fb0d5 2024-11-14T09:13:53,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:53,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575693876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:53,878 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:53,878 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:53,878 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:53,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:53,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575693876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:53,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:53,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:53,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:53,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/776de273b5d441c89ee4d88c9ed9c667 is 50, key is test_row_0/B:col10/1731575633436/Put/seqid=0 2024-11-14T09:13:53,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741877_1053 (size=12301) 2024-11-14T09:13:53,901 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=508 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/776de273b5d441c89ee4d88c9ed9c667 2024-11-14T09:13:53,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/f5089ca6fd9e424199d97f3e8d54bea3 is 50, key is test_row_0/C:col10/1731575633436/Put/seqid=0 2024-11-14T09:13:53,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741878_1054 (size=12301) 2024-11-14T09:13:53,937 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.02 KB at sequenceid=508 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/f5089ca6fd9e424199d97f3e8d54bea3 2024-11-14T09:13:53,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/93d10d3b98154fdfa4e8e43a0d8fb0d5 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/93d10d3b98154fdfa4e8e43a0d8fb0d5 2024-11-14T09:13:53,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:53,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49966 deadline: 1731575693950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:53,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:53,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:53,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:53,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:53,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49996 deadline: 1731575693950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:53,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:53,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:53,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:53,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:53,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49980 deadline: 1731575693950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:53,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:53,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:53,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:53,963 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/93d10d3b98154fdfa4e8e43a0d8fb0d5, entries=250, sequenceid=508, filesize=16.8 K 2024-11-14T09:13:53,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/776de273b5d441c89ee4d88c9ed9c667 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/776de273b5d441c89ee4d88c9ed9c667 2024-11-14T09:13:53,972 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/776de273b5d441c89ee4d88c9ed9c667, entries=150, sequenceid=508, filesize=12.0 K 2024-11-14T09:13:53,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/f5089ca6fd9e424199d97f3e8d54bea3 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/f5089ca6fd9e424199d97f3e8d54bea3 2024-11-14T09:13:53,985 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/f5089ca6fd9e424199d97f3e8d54bea3, entries=150, sequenceid=508, filesize=12.0 K 2024-11-14T09:13:53,987 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~39.86 KB/40815, heapSize ~105.12 KB/107640, currentSize=147.60 KB/151140 for a92123abacf02fce4f65c3ba94256e83 in 549ms, sequenceid=508, compaction requested=true 2024-11-14T09:13:53,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a92123abacf02fce4f65c3ba94256e83: 2024-11-14T09:13:53,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a92123abacf02fce4f65c3ba94256e83:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:13:53,988 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:13:53,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:13:53,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a92123abacf02fce4f65c3ba94256e83:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:13:53,988 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:13:53,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:13:53,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a92123abacf02fce4f65c3ba94256e83:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:13:53,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:13:53,990 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 47039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:13:53,990 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): a92123abacf02fce4f65c3ba94256e83/A is initiating minor compaction (all files) 2024-11-14T09:13:53,990 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a92123abacf02fce4f65c3ba94256e83/A in TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:53,990 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/1ef08393e3284940ab6e0c74c5403042, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/41f9beb335a24305902272a41cb276d1, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/93d10d3b98154fdfa4e8e43a0d8fb0d5] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp, totalSize=45.9 K 2024-11-14T09:13:53,991 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37279 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:13:53,991 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): a92123abacf02fce4f65c3ba94256e83/B is initiating minor compaction (all files) 2024-11-14T09:13:53,991 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a92123abacf02fce4f65c3ba94256e83/B in TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:53,991 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/a1ca9c7f90b6432f836343497117e2ca, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/6770f19af96c42de8198cef9aaa3af4c, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/776de273b5d441c89ee4d88c9ed9c667] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp, totalSize=36.4 K 2024-11-14T09:13:53,991 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ef08393e3284940ab6e0c74c5403042, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1731575632639 2024-11-14T09:13:53,991 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting a1ca9c7f90b6432f836343497117e2ca, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1731575632639 2024-11-14T09:13:53,992 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41f9beb335a24305902272a41cb276d1, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1731575632904 2024-11-14T09:13:53,992 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 6770f19af96c42de8198cef9aaa3af4c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1731575632904 2024-11-14T09:13:53,992 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93d10d3b98154fdfa4e8e43a0d8fb0d5, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=508, earliestPutTs=1731575633098 2024-11-14T09:13:53,993 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 776de273b5d441c89ee4d88c9ed9c667, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=508, earliestPutTs=1731575633434 2024-11-14T09:13:54,015 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a92123abacf02fce4f65c3ba94256e83#A#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:13:54,016 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/5345edf9034d4c1495348e1c41d7a839 is 50, key is test_row_0/A:col10/1731575633436/Put/seqid=0 2024-11-14T09:13:54,019 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a92123abacf02fce4f65c3ba94256e83#B#compaction#43 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:13:54,019 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/8e94fe9abb0d4cd98fd121a4c1b5f2c7 is 50, key is test_row_0/B:col10/1731575633436/Put/seqid=0 2024-11-14T09:13:54,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741880_1056 (size=12779) 2024-11-14T09:13:54,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741879_1055 (size=12779) 2024-11-14T09:13:54,059 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/5345edf9034d4c1495348e1c41d7a839 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/5345edf9034d4c1495348e1c41d7a839 2024-11-14T09:13:54,082 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a92123abacf02fce4f65c3ba94256e83/A of a92123abacf02fce4f65c3ba94256e83 into 5345edf9034d4c1495348e1c41d7a839(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:13:54,082 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a92123abacf02fce4f65c3ba94256e83: 2024-11-14T09:13:54,082 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., storeName=a92123abacf02fce4f65c3ba94256e83/A, priority=13, startTime=1731575633987; duration=0sec 2024-11-14T09:13:54,082 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:13:54,082 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a92123abacf02fce4f65c3ba94256e83:A 2024-11-14T09:13:54,082 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:13:54,084 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37279 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:13:54,084 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): a92123abacf02fce4f65c3ba94256e83/C is initiating minor compaction (all files) 2024-11-14T09:13:54,085 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a92123abacf02fce4f65c3ba94256e83/C in TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:54,085 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/19f0b5abff5344239b2d88b38d790861, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/5fcc5458fae24cb3884544764d1e8421, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/f5089ca6fd9e424199d97f3e8d54bea3] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp, totalSize=36.4 K 2024-11-14T09:13:54,086 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19f0b5abff5344239b2d88b38d790861, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1731575632639 2024-11-14T09:13:54,086 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fcc5458fae24cb3884544764d1e8421, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1731575632904 2024-11-14T09:13:54,087 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5089ca6fd9e424199d97f3e8d54bea3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=508, earliestPutTs=1731575633434 2024-11-14T09:13:54,098 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a92123abacf02fce4f65c3ba94256e83#C#compaction#44 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:13:54,099 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/0223fb546a4f495fbcf4c159d5ad89c2 is 50, key is test_row_0/C:col10/1731575633436/Put/seqid=0 2024-11-14T09:13:54,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741881_1057 (size=12779) 2024-11-14T09:13:54,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on a92123abacf02fce4f65c3ba94256e83 2024-11-14T09:13:54,193 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a92123abacf02fce4f65c3ba94256e83 3/3 column families, dataSize=154.31 KB heapSize=429.84 KB 2024-11-14T09:13:54,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=A 2024-11-14T09:13:54,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:13:54,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=B 2024-11-14T09:13:54,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:13:54,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=C 2024-11-14T09:13:54,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:13:54,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/d679062c9b5847b88ec3026d0c854385 is 50, key is test_row_0/A:col10/1731575633615/Put/seqid=0 2024-11-14T09:13:54,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:54,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575694201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:54,211 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:54,211 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:54,211 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:54,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:54,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575694204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:54,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:54,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:54,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:54,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741882_1058 (size=14741) 2024-11-14T09:13:54,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T09:13:54,265 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:13:54,266 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:13:54,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-14T09:13:54,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:54,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49996 deadline: 1731575694265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:54,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:54,270 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:13:54,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:54,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:54,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:54,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:54,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49980 deadline: 1731575694265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:54,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49966 deadline: 1731575694265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:54,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:54,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:54,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:54,271 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:54,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:54,271 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:54,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T09:13:54,271 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:13:54,271 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:13:54,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:54,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575694315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:54,318 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:54,319 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:54,319 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:54,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:54,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575694325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:54,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:54,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:54,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:54,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T09:13:54,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T09:13:54,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:54,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:13:54,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:54,426 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:54,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:54,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:54,454 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/8e94fe9abb0d4cd98fd121a4c1b5f2c7 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/8e94fe9abb0d4cd98fd121a4c1b5f2c7 2024-11-14T09:13:54,465 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a92123abacf02fce4f65c3ba94256e83/B of a92123abacf02fce4f65c3ba94256e83 into 8e94fe9abb0d4cd98fd121a4c1b5f2c7(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:13:54,465 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a92123abacf02fce4f65c3ba94256e83: 2024-11-14T09:13:54,465 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., storeName=a92123abacf02fce4f65c3ba94256e83/B, priority=13, startTime=1731575633988; duration=0sec 2024-11-14T09:13:54,465 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:13:54,465 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a92123abacf02fce4f65c3ba94256e83:B 2024-11-14T09:13:54,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:54,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575694525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:54,529 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:54,529 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:54,529 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:54,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:54,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575694535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:54,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:54,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:54,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:54,548 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/0223fb546a4f495fbcf4c159d5ad89c2 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/0223fb546a4f495fbcf4c159d5ad89c2 2024-11-14T09:13:54,555 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a92123abacf02fce4f65c3ba94256e83/C of a92123abacf02fce4f65c3ba94256e83 into 0223fb546a4f495fbcf4c159d5ad89c2(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:13:54,555 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a92123abacf02fce4f65c3ba94256e83: 2024-11-14T09:13:54,555 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., storeName=a92123abacf02fce4f65c3ba94256e83/C, priority=13, startTime=1731575633988; duration=0sec 2024-11-14T09:13:54,556 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:13:54,556 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a92123abacf02fce4f65c3ba94256e83:C 2024-11-14T09:13:54,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T09:13:54,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:54,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:13:54,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:54,581 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:54,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:54,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:54,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T09:13:54,620 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=564 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/d679062c9b5847b88ec3026d0c854385 2024-11-14T09:13:54,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/db6c2237c6064af6872c8ebd158193c7 is 50, key is test_row_0/B:col10/1731575633615/Put/seqid=0 2024-11-14T09:13:54,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741883_1059 (size=12301) 2024-11-14T09:13:54,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T09:13:54,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:54,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:13:54,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:54,736 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:54,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:54,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:54,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:54,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49996 deadline: 1731575694775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:54,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:54,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49966 deadline: 1731575694776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:54,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:54,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:54,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:54,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:54,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49980 deadline: 1731575694776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:54,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:54,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:54,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:54,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:54,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:54,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:54,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:54,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575694835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:54,840 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:54,840 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:54,840 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:54,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:54,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575694845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:54,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:54,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:54,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:54,890 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T09:13:54,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:54,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:13:54,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:54,890 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:54,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:54,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:54,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T09:13:55,043 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=564 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/db6c2237c6064af6872c8ebd158193c7 2024-11-14T09:13:55,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T09:13:55,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:55,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:13:55,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:55,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:55,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:55,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:55,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/fe1c391d3288430d8e1c3f2a31d451e6 is 50, key is test_row_0/C:col10/1731575633615/Put/seqid=0 2024-11-14T09:13:55,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741884_1060 (size=12301) 2024-11-14T09:13:55,072 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=564 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/fe1c391d3288430d8e1c3f2a31d451e6 2024-11-14T09:13:55,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/d679062c9b5847b88ec3026d0c854385 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/d679062c9b5847b88ec3026d0c854385 2024-11-14T09:13:55,107 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/d679062c9b5847b88ec3026d0c854385, entries=200, sequenceid=564, filesize=14.4 K 2024-11-14T09:13:55,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/db6c2237c6064af6872c8ebd158193c7 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/db6c2237c6064af6872c8ebd158193c7 2024-11-14T09:13:55,121 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/db6c2237c6064af6872c8ebd158193c7, entries=150, sequenceid=564, filesize=12.0 K 2024-11-14T09:13:55,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/fe1c391d3288430d8e1c3f2a31d451e6 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/fe1c391d3288430d8e1c3f2a31d451e6 2024-11-14T09:13:55,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/fe1c391d3288430d8e1c3f2a31d451e6, entries=150, sequenceid=564, filesize=12.0 K 2024-11-14T09:13:55,136 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~429.80 KB/440112, currentSize=33.54 KB/34350 for a92123abacf02fce4f65c3ba94256e83 in 942ms, sequenceid=564, compaction requested=false 2024-11-14T09:13:55,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a92123abacf02fce4f65c3ba94256e83: 2024-11-14T09:13:55,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T09:13:55,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:55,200 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing a92123abacf02fce4f65c3ba94256e83 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-14T09:13:55,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=A 2024-11-14T09:13:55,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:13:55,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=B 2024-11-14T09:13:55,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:13:55,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=C 2024-11-14T09:13:55,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:13:55,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/e528b16185a44ed8b4eeeaa5fbe398e8 is 50, key is test_row_0/A:col10/1731575634201/Put/seqid=0 2024-11-14T09:13:55,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741885_1061 (size=12301) 2024-11-14T09:13:55,221 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=574 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/e528b16185a44ed8b4eeeaa5fbe398e8 2024-11-14T09:13:55,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/6f26b8f249f24341bea45680bb9aeca0 is 50, key is test_row_0/B:col10/1731575634201/Put/seqid=0 2024-11-14T09:13:55,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741886_1062 (size=12301) 2024-11-14T09:13:55,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on a92123abacf02fce4f65c3ba94256e83 2024-11-14T09:13:55,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:13:55,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T09:13:55,410 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731575635394, max timestamp=1731575635404Num uniques 859; 2024-11-14T09:13:55,410 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:13:55,413 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731575635394, max timestamp=1731575635407Num uniques 859; 2024-11-14T09:13:55,413 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:13:55,415 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731575635394, max timestamp=1731575635404Num uniques 859; 2024-11-14T09:13:55,415 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-14T09:13:55,483 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731575635422, max timestamp=1731575635478Num uniques 1018; 2024-11-14T09:13:55,483 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-14T09:13:55,484 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731575635422, max timestamp=1731575635478Num uniques 1018; 2024-11-14T09:13:55,484 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-14T09:13:55,488 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731575635422, max timestamp=1731575635478Num uniques 1009; 2024-11-14T09:13:55,488 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=12268, suffix heap size=132904,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=268 2024-11-14T09:13:55,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:55,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575695538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:55,540 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:55,540 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:55,540 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:55,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:55,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575695539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:55,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:55,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:55,547 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:55,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:55,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575695645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:55,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:55,649 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=574 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/6f26b8f249f24341bea45680bb9aeca0 2024-11-14T09:13:55,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:55,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:55,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:55,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575695655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:55,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:55,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:55,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:55,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/09d3ef601e3a448d8dfd5d19ff515880 is 50, key is test_row_0/C:col10/1731575634201/Put/seqid=0 2024-11-14T09:13:55,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741887_1063 (size=12301) 2024-11-14T09:13:55,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:55,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49996 deadline: 1731575695785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:55,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:55,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49980 deadline: 1731575695785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:55,795 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_1' in a92123abacf02fce4f65c3ba94256e83 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:13:55,795 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_1' in a92123abacf02fce4f65c3ba94256e83 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:13:55,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:55,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:55,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:55,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:55,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:55,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:55,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:55,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49966 deadline: 1731575695795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:55,799 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_0' in a92123abacf02fce4f65c3ba94256e83 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:13:55,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:55,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:55,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:55,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:55,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575695855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:55,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:55,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:55,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:55,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:55,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575695865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:55,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:55,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:55,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:56,067 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=574 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/09d3ef601e3a448d8dfd5d19ff515880 2024-11-14T09:13:56,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/e528b16185a44ed8b4eeeaa5fbe398e8 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/e528b16185a44ed8b4eeeaa5fbe398e8 2024-11-14T09:13:56,087 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/e528b16185a44ed8b4eeeaa5fbe398e8, entries=150, sequenceid=574, filesize=12.0 K 2024-11-14T09:13:56,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/6f26b8f249f24341bea45680bb9aeca0 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/6f26b8f249f24341bea45680bb9aeca0 2024-11-14T09:13:56,096 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/6f26b8f249f24341bea45680bb9aeca0, entries=150, sequenceid=574, filesize=12.0 K 2024-11-14T09:13:56,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/09d3ef601e3a448d8dfd5d19ff515880 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/09d3ef601e3a448d8dfd5d19ff515880 2024-11-14T09:13:56,106 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/09d3ef601e3a448d8dfd5d19ff515880, entries=150, sequenceid=574, filesize=12.0 K 2024-11-14T09:13:56,107 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=147.60 KB/151140 for a92123abacf02fce4f65c3ba94256e83 in 907ms, sequenceid=574, compaction requested=true 2024-11-14T09:13:56,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for a92123abacf02fce4f65c3ba94256e83: 2024-11-14T09:13:56,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:56,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-14T09:13:56,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-14T09:13:56,111 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-14T09:13:56,111 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8380 sec 2024-11-14T09:13:56,117 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.8490 sec 2024-11-14T09:13:56,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on a92123abacf02fce4f65c3ba94256e83 2024-11-14T09:13:56,168 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a92123abacf02fce4f65c3ba94256e83 3/3 column families, dataSize=154.31 KB heapSize=435.23 KB 2024-11-14T09:13:56,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=A 2024-11-14T09:13:56,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-14T09:13:56,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=B 2024-11-14T09:13:56,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-14T09:13:56,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=C 2024-11-14T09:13:56,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-14T09:13:56,177 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/14fdd7793cda4a36a5d18f42447d52c2 is 50, key is test_row_0/A:col10/1731575635522/Put/seqid=0 2024-11-14T09:13:56,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:56,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575696182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:56,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741888_1064 (size=14741) 2024-11-14T09:13:56,186 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:56,186 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:56,186 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:56,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:56,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575696185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:56,187 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:56,187 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:56,187 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:56,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:56,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575696296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:56,302 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:56,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:56,303 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:56,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575696298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:56,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:56,304 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:56,304 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:56,303 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:56,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T09:13:56,415 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:13:56,416 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:13:56,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-14T09:13:56,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T09:13:56,418 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:13:56,419 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:13:56,419 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:13:56,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:56,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:56,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575696515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:56,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575696515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:56,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:56,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:56,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:56,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:56,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:56,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:56,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T09:13:56,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-14T09:13:56,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:56,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:13:56,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:56,574 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:56,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:56,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:56,587 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=629 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/14fdd7793cda4a36a5d18f42447d52c2 2024-11-14T09:13:56,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/e3c814027e47479488c625531b0be89a is 50, key is test_row_0/B:col10/1731575635522/Put/seqid=0 2024-11-14T09:13:56,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741889_1065 (size=12301) 2024-11-14T09:13:56,617 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=629 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/e3c814027e47479488c625531b0be89a 2024-11-14T09:13:56,632 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/4988ffdf37af48f3b23acd289a522449 is 50, key is test_row_0/C:col10/1731575635522/Put/seqid=0 2024-11-14T09:13:56,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741890_1066 (size=12301) 2024-11-14T09:13:56,641 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=629 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/4988ffdf37af48f3b23acd289a522449 2024-11-14T09:13:56,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/14fdd7793cda4a36a5d18f42447d52c2 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/14fdd7793cda4a36a5d18f42447d52c2 2024-11-14T09:13:56,663 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/14fdd7793cda4a36a5d18f42447d52c2, entries=200, sequenceid=629, filesize=14.4 K 2024-11-14T09:13:56,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/e3c814027e47479488c625531b0be89a as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/e3c814027e47479488c625531b0be89a 2024-11-14T09:13:56,673 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/e3c814027e47479488c625531b0be89a, entries=150, sequenceid=629, filesize=12.0 K 2024-11-14T09:13:56,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/4988ffdf37af48f3b23acd289a522449 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/4988ffdf37af48f3b23acd289a522449 2024-11-14T09:13:56,688 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/4988ffdf37af48f3b23acd289a522449, entries=150, sequenceid=629, filesize=12.0 K 2024-11-14T09:13:56,689 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~435.19 KB/445632, currentSize=26.84 KB/27480 for a92123abacf02fce4f65c3ba94256e83 in 521ms, sequenceid=629, compaction requested=true 2024-11-14T09:13:56,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a92123abacf02fce4f65c3ba94256e83: 2024-11-14T09:13:56,689 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:13:56,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a92123abacf02fce4f65c3ba94256e83:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:13:56,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:13:56,690 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:13:56,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a92123abacf02fce4f65c3ba94256e83:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:13:56,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:13:56,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a92123abacf02fce4f65c3ba94256e83:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:13:56,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:13:56,693 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54562 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:13:56,693 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): a92123abacf02fce4f65c3ba94256e83/A is initiating minor compaction (all files) 2024-11-14T09:13:56,693 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a92123abacf02fce4f65c3ba94256e83/A in TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:56,693 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/5345edf9034d4c1495348e1c41d7a839, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/d679062c9b5847b88ec3026d0c854385, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/e528b16185a44ed8b4eeeaa5fbe398e8, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/14fdd7793cda4a36a5d18f42447d52c2] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp, totalSize=53.3 K 2024-11-14T09:13:56,694 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49682 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:13:56,694 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): a92123abacf02fce4f65c3ba94256e83/B is initiating minor compaction (all files) 2024-11-14T09:13:56,694 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a92123abacf02fce4f65c3ba94256e83/B in TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:56,694 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/8e94fe9abb0d4cd98fd121a4c1b5f2c7, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/db6c2237c6064af6872c8ebd158193c7, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/6f26b8f249f24341bea45680bb9aeca0, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/e3c814027e47479488c625531b0be89a] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp, totalSize=48.5 K 2024-11-14T09:13:56,694 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5345edf9034d4c1495348e1c41d7a839, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=508, earliestPutTs=1731575633434 2024-11-14T09:13:56,695 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e94fe9abb0d4cd98fd121a4c1b5f2c7, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=508, earliestPutTs=1731575633434 2024-11-14T09:13:56,695 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting d679062c9b5847b88ec3026d0c854385, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=564, earliestPutTs=1731575633609 2024-11-14T09:13:56,697 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting db6c2237c6064af6872c8ebd158193c7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=564, earliestPutTs=1731575633609 2024-11-14T09:13:56,698 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f26b8f249f24341bea45680bb9aeca0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=574, earliestPutTs=1731575634196 2024-11-14T09:13:56,699 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting e528b16185a44ed8b4eeeaa5fbe398e8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=574, earliestPutTs=1731575634196 2024-11-14T09:13:56,700 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14fdd7793cda4a36a5d18f42447d52c2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=629, earliestPutTs=1731575635522 2024-11-14T09:13:56,700 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting e3c814027e47479488c625531b0be89a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=629, earliestPutTs=1731575635522 2024-11-14T09:13:56,724 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a92123abacf02fce4f65c3ba94256e83#A#compaction#54 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:13:56,724 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/b18068da669940cd99ba9435673f3b02 is 50, key is test_row_0/A:col10/1731575635522/Put/seqid=0 2024-11-14T09:13:56,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-14T09:13:56,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:56,729 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing a92123abacf02fce4f65c3ba94256e83 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-14T09:13:56,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=A 2024-11-14T09:13:56,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:13:56,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=B 2024-11-14T09:13:56,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:13:56,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=C 2024-11-14T09:13:56,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:13:56,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T09:13:56,740 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a92123abacf02fce4f65c3ba94256e83#B#compaction#55 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:13:56,741 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/65bc053b0c49432484d497989ec5e158 is 50, key is test_row_0/B:col10/1731575635522/Put/seqid=0 2024-11-14T09:13:56,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/f22c370481564b6a88e5bb3387b9ae4f is 50, key is test_row_0/A:col10/1731575636180/Put/seqid=0 2024-11-14T09:13:56,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741891_1067 (size=12915) 2024-11-14T09:13:56,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741893_1069 (size=9857) 2024-11-14T09:13:56,763 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=636 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/f22c370481564b6a88e5bb3387b9ae4f 2024-11-14T09:13:56,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741892_1068 (size=12915) 2024-11-14T09:13:56,773 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/65bc053b0c49432484d497989ec5e158 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/65bc053b0c49432484d497989ec5e158 2024-11-14T09:13:56,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/b9eb900add4142189ff8f6be0fc0f53e is 50, key is test_row_0/B:col10/1731575636180/Put/seqid=0 2024-11-14T09:13:56,795 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a92123abacf02fce4f65c3ba94256e83/B of a92123abacf02fce4f65c3ba94256e83 into 65bc053b0c49432484d497989ec5e158(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:13:56,795 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a92123abacf02fce4f65c3ba94256e83: 2024-11-14T09:13:56,795 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., storeName=a92123abacf02fce4f65c3ba94256e83/B, priority=12, startTime=1731575636690; duration=0sec 2024-11-14T09:13:56,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741894_1070 (size=9857) 2024-11-14T09:13:56,796 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:13:56,796 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a92123abacf02fce4f65c3ba94256e83:B 2024-11-14T09:13:56,796 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:13:56,797 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=636 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/b9eb900add4142189ff8f6be0fc0f53e 2024-11-14T09:13:56,800 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49682 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:13:56,800 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): a92123abacf02fce4f65c3ba94256e83/C is initiating minor compaction (all files) 2024-11-14T09:13:56,800 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a92123abacf02fce4f65c3ba94256e83/C in TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:56,800 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/0223fb546a4f495fbcf4c159d5ad89c2, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/fe1c391d3288430d8e1c3f2a31d451e6, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/09d3ef601e3a448d8dfd5d19ff515880, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/4988ffdf37af48f3b23acd289a522449] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp, totalSize=48.5 K 2024-11-14T09:13:56,804 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 0223fb546a4f495fbcf4c159d5ad89c2, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=508, earliestPutTs=1731575633434 2024-11-14T09:13:56,805 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting fe1c391d3288430d8e1c3f2a31d451e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=564, earliestPutTs=1731575633609 2024-11-14T09:13:56,806 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 09d3ef601e3a448d8dfd5d19ff515880, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=574, earliestPutTs=1731575634196 2024-11-14T09:13:56,807 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 4988ffdf37af48f3b23acd289a522449, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=629, earliestPutTs=1731575635522 2024-11-14T09:13:56,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/644a522b98d24fe7a1367d430c9dc738 is 50, key is test_row_0/C:col10/1731575636180/Put/seqid=0 2024-11-14T09:13:56,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741895_1071 (size=9857) 2024-11-14T09:13:56,833 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a92123abacf02fce4f65c3ba94256e83#C#compaction#59 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:13:56,834 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/4353b58396794073802ea808c8421935 is 50, key is test_row_0/C:col10/1731575635522/Put/seqid=0 2024-11-14T09:13:56,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on a92123abacf02fce4f65c3ba94256e83 2024-11-14T09:13:56,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:13:56,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741896_1072 (size=12915) 2024-11-14T09:13:56,871 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/4353b58396794073802ea808c8421935 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/4353b58396794073802ea808c8421935 2024-11-14T09:13:56,879 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a92123abacf02fce4f65c3ba94256e83/C of a92123abacf02fce4f65c3ba94256e83 into 4353b58396794073802ea808c8421935(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:13:56,879 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a92123abacf02fce4f65c3ba94256e83: 2024-11-14T09:13:56,879 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., storeName=a92123abacf02fce4f65c3ba94256e83/C, priority=12, startTime=1731575636690; duration=0sec 2024-11-14T09:13:56,879 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:13:56,879 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a92123abacf02fce4f65c3ba94256e83:C 2024-11-14T09:13:56,893 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731575636863, max timestamp=1731575636889Num uniques 859; 2024-11-14T09:13:56,893 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:13:56,894 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731575636863, max timestamp=1731575636889Num uniques 859; 2024-11-14T09:13:56,894 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:13:56,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731575636863, max timestamp=1731575636889Num uniques 859; 2024-11-14T09:13:56,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:13:56,951 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731575636936, max timestamp=1731575636946Num uniques 1018; 2024-11-14T09:13:56,951 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-14T09:13:56,951 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731575636936, max timestamp=1731575636946Num uniques 1018; 2024-11-14T09:13:56,952 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-14T09:13:56,955 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731575636936, max timestamp=1731575636946Num uniques 1018; 2024-11-14T09:13:56,955 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-14T09:13:57,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T09:13:57,049 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731575637000, max timestamp=1731575637043Num uniques 1027; 2024-11-14T09:13:57,049 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-14T09:13:57,049 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731575637000, max timestamp=1731575637043Num uniques 1027; 2024-11-14T09:13:57,049 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731575637000, max timestamp=1731575637043Num uniques 1027; 2024-11-14T09:13:57,049 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-14T09:13:57,049 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-14T09:13:57,120 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731575637085, max timestamp=1731575637115Num uniques 1036; 2024-11-14T09:13:57,120 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-14T09:13:57,121 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731575637085, max timestamp=1731575637115Num uniques 1036; 2024-11-14T09:13:57,121 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-14T09:13:57,120 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731575637085, max timestamp=1731575637115Num uniques 1036; 2024-11-14T09:13:57,121 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-14T09:13:57,164 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/b18068da669940cd99ba9435673f3b02 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/b18068da669940cd99ba9435673f3b02 2024-11-14T09:13:57,175 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a92123abacf02fce4f65c3ba94256e83/A of a92123abacf02fce4f65c3ba94256e83 into b18068da669940cd99ba9435673f3b02(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:13:57,175 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a92123abacf02fce4f65c3ba94256e83: 2024-11-14T09:13:57,175 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., storeName=a92123abacf02fce4f65c3ba94256e83/A, priority=12, startTime=1731575636689; duration=0sec 2024-11-14T09:13:57,175 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:13:57,175 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a92123abacf02fce4f65c3ba94256e83:A 2024-11-14T09:13:57,192 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1731575637137, max timestamp=1731575637187Num uniques 1095; 2024-11-14T09:13:57,192 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=15800, suffix heap size=141848,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=345 2024-11-14T09:13:57,193 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1731575637137, max timestamp=1731575637187Num uniques 1095; 2024-11-14T09:13:57,193 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=15800, suffix heap size=141848,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=345 2024-11-14T09:13:57,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731575637137, max timestamp=1731575637187Num uniques 1095; 2024-11-14T09:13:57,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-14T09:13:57,225 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=636 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/644a522b98d24fe7a1367d430c9dc738 2024-11-14T09:13:57,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/f22c370481564b6a88e5bb3387b9ae4f as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/f22c370481564b6a88e5bb3387b9ae4f 2024-11-14T09:13:57,241 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/f22c370481564b6a88e5bb3387b9ae4f, entries=100, sequenceid=636, filesize=9.6 K 2024-11-14T09:13:57,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/b9eb900add4142189ff8f6be0fc0f53e as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/b9eb900add4142189ff8f6be0fc0f53e 2024-11-14T09:13:57,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:57,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575697238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:57,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:57,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:57,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:57,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:57,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575697244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:57,249 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:57,249 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:57,249 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:57,252 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/b9eb900add4142189ff8f6be0fc0f53e, entries=100, sequenceid=636, filesize=9.6 K 2024-11-14T09:13:57,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/644a522b98d24fe7a1367d430c9dc738 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/644a522b98d24fe7a1367d430c9dc738 2024-11-14T09:13:57,263 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/644a522b98d24fe7a1367d430c9dc738, entries=100, sequenceid=636, filesize=9.6 K 2024-11-14T09:13:57,264 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=156.54 KB/160300 for a92123abacf02fce4f65c3ba94256e83 in 535ms, sequenceid=636, compaction requested=false 2024-11-14T09:13:57,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for a92123abacf02fce4f65c3ba94256e83: 2024-11-14T09:13:57,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:57,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-14T09:13:57,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-14T09:13:57,268 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-14T09:13:57,268 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 847 msec 2024-11-14T09:13:57,271 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 853 msec 2024-11-14T09:13:57,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on a92123abacf02fce4f65c3ba94256e83 2024-11-14T09:13:57,362 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a92123abacf02fce4f65c3ba94256e83 3/3 column families, dataSize=163.25 KB heapSize=457.70 KB 2024-11-14T09:13:57,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=A 2024-11-14T09:13:57,363 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731575637228, max timestamp=1731575637358Num uniques 1204; 2024-11-14T09:13:57,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-14T09:13:57,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=B 2024-11-14T09:13:57,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:13:57,363 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=55140, new segment data size=7050, suffix heap size=153184,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1204, new segment cells count=154 2024-11-14T09:13:57,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=C 2024-11-14T09:13:57,363 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731575637228, max timestamp=1731575637358Num uniques 1104; 2024-11-14T09:13:57,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:13:57,363 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-11-14T09:13:57,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/8afdaad81e414b98b9f75c4a44cb7b6a is 50, key is test_row_0/A:col10/1731575637358/Put/seqid=0 2024-11-14T09:13:57,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741897_1073 (size=17181) 2024-11-14T09:13:57,381 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=747 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/8afdaad81e414b98b9f75c4a44cb7b6a 2024-11-14T09:13:57,392 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/bde88938d36a44e7b75761ceb01de4e3 is 50, key is test_row_0/B:col10/1731575637358/Put/seqid=0 2024-11-14T09:13:57,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741898_1074 (size=12301) 2024-11-14T09:13:57,404 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=747 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/bde88938d36a44e7b75761ceb01de4e3 2024-11-14T09:13:57,417 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/4c53cf2a372547cf9720c5335ae34ac4 is 50, key is test_row_0/C:col10/1731575637358/Put/seqid=0 2024-11-14T09:13:57,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741899_1075 (size=12301) 2024-11-14T09:13:57,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:57,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575697420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:57,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:57,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:57,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575697422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:57,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:57,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:57,427 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:57,427 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:57,427 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:57,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:57,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575697534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:57,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:57,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575697535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:57,543 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:57,543 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:57,543 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:57,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:57,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:57,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:57,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T09:13:57,555 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:13:57,556 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:13:57,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-14T09:13:57,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-14T09:13:57,558 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:13:57,558 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:13:57,558 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:13:57,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-14T09:13:57,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-14T09:13:57,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:57,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:13:57,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:57,711 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:57,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:57,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:57,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:57,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575697755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:57,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:57,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:57,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:57,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:57,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575697755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:57,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:57,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:57,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:57,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:57,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49980 deadline: 1731575697814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:57,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:57,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49996 deadline: 1731575697815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:57,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:57,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49966 deadline: 1731575697815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:57,816 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_1' in a92123abacf02fce4f65c3ba94256e83 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:13:57,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:57,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:57,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:57,817 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_0' in a92123abacf02fce4f65c3ba94256e83 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:13:57,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:57,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:57,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:57,818 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_1' in a92123abacf02fce4f65c3ba94256e83 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:13:57,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:57,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:57,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:57,827 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=747 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/4c53cf2a372547cf9720c5335ae34ac4 2024-11-14T09:13:57,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/8afdaad81e414b98b9f75c4a44cb7b6a as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/8afdaad81e414b98b9f75c4a44cb7b6a 2024-11-14T09:13:57,840 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/8afdaad81e414b98b9f75c4a44cb7b6a, entries=250, sequenceid=747, filesize=16.8 K 2024-11-14T09:13:57,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/bde88938d36a44e7b75761ceb01de4e3 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/bde88938d36a44e7b75761ceb01de4e3 2024-11-14T09:13:57,848 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/bde88938d36a44e7b75761ceb01de4e3, entries=150, sequenceid=747, filesize=12.0 K 2024-11-14T09:13:57,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/4c53cf2a372547cf9720c5335ae34ac4 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/4c53cf2a372547cf9720c5335ae34ac4 2024-11-14T09:13:57,857 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/4c53cf2a372547cf9720c5335ae34ac4, entries=150, sequenceid=747, filesize=12.0 K 2024-11-14T09:13:57,858 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~220.30 KB/225584, currentSize=107.34 KB/109920 for a92123abacf02fce4f65c3ba94256e83 in 496ms, sequenceid=747, compaction requested=true 2024-11-14T09:13:57,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a92123abacf02fce4f65c3ba94256e83: 2024-11-14T09:13:57,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a92123abacf02fce4f65c3ba94256e83:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:13:57,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:13:57,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a92123abacf02fce4f65c3ba94256e83:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:13:57,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:13:57,858 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:13:57,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a92123abacf02fce4f65c3ba94256e83:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:13:57,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:13:57,858 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:13:57,860 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39953 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:13:57,860 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:13:57,860 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): a92123abacf02fce4f65c3ba94256e83/A is initiating minor compaction (all files) 2024-11-14T09:13:57,860 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): a92123abacf02fce4f65c3ba94256e83/B is initiating minor compaction (all files) 2024-11-14T09:13:57,860 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a92123abacf02fce4f65c3ba94256e83/A in TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:57,860 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a92123abacf02fce4f65c3ba94256e83/B in TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:57,860 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/b18068da669940cd99ba9435673f3b02, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/f22c370481564b6a88e5bb3387b9ae4f, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/8afdaad81e414b98b9f75c4a44cb7b6a] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp, totalSize=39.0 K 2024-11-14T09:13:57,860 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/65bc053b0c49432484d497989ec5e158, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/b9eb900add4142189ff8f6be0fc0f53e, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/bde88938d36a44e7b75761ceb01de4e3] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp, totalSize=34.3 K 2024-11-14T09:13:57,861 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting b18068da669940cd99ba9435673f3b02, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=629, earliestPutTs=1731575635522 2024-11-14T09:13:57,861 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 65bc053b0c49432484d497989ec5e158, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=629, earliestPutTs=1731575635522 2024-11-14T09:13:57,861 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting b9eb900add4142189ff8f6be0fc0f53e, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=636, earliestPutTs=1731575636180 2024-11-14T09:13:57,861 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting f22c370481564b6a88e5bb3387b9ae4f, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=636, earliestPutTs=1731575636180 2024-11-14T09:13:57,862 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting bde88938d36a44e7b75761ceb01de4e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=747, earliestPutTs=1731575637228 2024-11-14T09:13:57,862 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8afdaad81e414b98b9f75c4a44cb7b6a, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=747, earliestPutTs=1731575637228 2024-11-14T09:13:57,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-14T09:13:57,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:57,865 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing a92123abacf02fce4f65c3ba94256e83 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-14T09:13:57,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=A 2024-11-14T09:13:57,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:13:57,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=B 2024-11-14T09:13:57,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:13:57,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=C 2024-11-14T09:13:57,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:13:57,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/d1ed19fcc97b45829d8cd7bfacee640f is 50, key is test_row_0/A:col10/1731575637418/Put/seqid=0 2024-11-14T09:13:57,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-14T09:13:57,882 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a92123abacf02fce4f65c3ba94256e83#B#compaction#64 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:13:57,882 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a92123abacf02fce4f65c3ba94256e83#A#compaction#65 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:13:57,883 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/2a83db7c6d144326a946f31abaedd4fb is 50, key is test_row_0/B:col10/1731575637358/Put/seqid=0 2024-11-14T09:13:57,883 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/96ea282569d94124b9bf0cfa01b17e5b is 50, key is test_row_0/A:col10/1731575637358/Put/seqid=0 2024-11-14T09:13:57,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741901_1077 (size=13017) 2024-11-14T09:13:57,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741900_1076 (size=12301) 2024-11-14T09:13:57,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741902_1078 (size=13017) 2024-11-14T09:13:58,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on a92123abacf02fce4f65c3ba94256e83 2024-11-14T09:13:58,067 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:13:58,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:58,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575698085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:58,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:58,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:58,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:58,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:58,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575698091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:58,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:58,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:58,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:58,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-14T09:13:58,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:58,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575698195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:58,206 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:58,207 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:58,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:58,207 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:58,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575698205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:58,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:58,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:58,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:58,289 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=766 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/d1ed19fcc97b45829d8cd7bfacee640f 2024-11-14T09:13:58,304 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/2a83db7c6d144326a946f31abaedd4fb as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/2a83db7c6d144326a946f31abaedd4fb 2024-11-14T09:13:58,306 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/96ea282569d94124b9bf0cfa01b17e5b as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/96ea282569d94124b9bf0cfa01b17e5b 2024-11-14T09:13:58,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/38285b08250d4fe8a5aa7c6ce77227e9 is 50, key is test_row_0/B:col10/1731575637418/Put/seqid=0 2024-11-14T09:13:58,322 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a92123abacf02fce4f65c3ba94256e83/A of a92123abacf02fce4f65c3ba94256e83 into 96ea282569d94124b9bf0cfa01b17e5b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:13:58,322 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a92123abacf02fce4f65c3ba94256e83: 2024-11-14T09:13:58,322 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., storeName=a92123abacf02fce4f65c3ba94256e83/A, priority=13, startTime=1731575637858; duration=0sec 2024-11-14T09:13:58,323 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:13:58,323 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a92123abacf02fce4f65c3ba94256e83:A 2024-11-14T09:13:58,323 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:13:58,324 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:13:58,324 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): a92123abacf02fce4f65c3ba94256e83/C is initiating minor compaction (all files) 2024-11-14T09:13:58,324 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a92123abacf02fce4f65c3ba94256e83/C in TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:58,324 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/4353b58396794073802ea808c8421935, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/644a522b98d24fe7a1367d430c9dc738, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/4c53cf2a372547cf9720c5335ae34ac4] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp, totalSize=34.3 K 2024-11-14T09:13:58,327 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a92123abacf02fce4f65c3ba94256e83/B of a92123abacf02fce4f65c3ba94256e83 into 2a83db7c6d144326a946f31abaedd4fb(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:13:58,327 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a92123abacf02fce4f65c3ba94256e83: 2024-11-14T09:13:58,327 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., storeName=a92123abacf02fce4f65c3ba94256e83/B, priority=13, startTime=1731575637858; duration=0sec 2024-11-14T09:13:58,327 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:13:58,327 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a92123abacf02fce4f65c3ba94256e83:B 2024-11-14T09:13:58,329 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4353b58396794073802ea808c8421935, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=629, earliestPutTs=1731575635522 2024-11-14T09:13:58,340 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 644a522b98d24fe7a1367d430c9dc738, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=636, earliestPutTs=1731575636180 2024-11-14T09:13:58,341 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c53cf2a372547cf9720c5335ae34ac4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=747, earliestPutTs=1731575637228 2024-11-14T09:13:58,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741903_1079 (size=12301) 2024-11-14T09:13:58,357 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=766 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/38285b08250d4fe8a5aa7c6ce77227e9 2024-11-14T09:13:58,364 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a92123abacf02fce4f65c3ba94256e83#C#compaction#67 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:13:58,365 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/9798e58624a94992808a8020bd535514 is 50, key is test_row_0/C:col10/1731575637358/Put/seqid=0 2024-11-14T09:13:58,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/9a8d65c63adf45b5bdfac78669c2378a is 50, key is test_row_0/C:col10/1731575637418/Put/seqid=0 2024-11-14T09:13:58,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741904_1080 (size=13017) 2024-11-14T09:13:58,388 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/9798e58624a94992808a8020bd535514 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/9798e58624a94992808a8020bd535514 2024-11-14T09:13:58,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741905_1081 (size=12301) 2024-11-14T09:13:58,392 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=766 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/9a8d65c63adf45b5bdfac78669c2378a 2024-11-14T09:13:58,396 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a92123abacf02fce4f65c3ba94256e83/C of a92123abacf02fce4f65c3ba94256e83 into 9798e58624a94992808a8020bd535514(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:13:58,396 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a92123abacf02fce4f65c3ba94256e83: 2024-11-14T09:13:58,396 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., storeName=a92123abacf02fce4f65c3ba94256e83/C, priority=13, startTime=1731575637858; duration=0sec 2024-11-14T09:13:58,396 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:13:58,396 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a92123abacf02fce4f65c3ba94256e83:C 2024-11-14T09:13:58,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/d1ed19fcc97b45829d8cd7bfacee640f as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/d1ed19fcc97b45829d8cd7bfacee640f 2024-11-14T09:13:58,404 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/d1ed19fcc97b45829d8cd7bfacee640f, entries=150, sequenceid=766, filesize=12.0 K 2024-11-14T09:13:58,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/38285b08250d4fe8a5aa7c6ce77227e9 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/38285b08250d4fe8a5aa7c6ce77227e9 2024-11-14T09:13:58,411 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/38285b08250d4fe8a5aa7c6ce77227e9, entries=150, sequenceid=766, filesize=12.0 K 2024-11-14T09:13:58,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/9a8d65c63adf45b5bdfac78669c2378a as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/9a8d65c63adf45b5bdfac78669c2378a 2024-11-14T09:13:58,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:58,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575698415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:58,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:58,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575698415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:58,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:58,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:58,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:58,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:58,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:58,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:58,423 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/9a8d65c63adf45b5bdfac78669c2378a, entries=150, sequenceid=766, filesize=12.0 K 2024-11-14T09:13:58,424 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for a92123abacf02fce4f65c3ba94256e83 in 559ms, sequenceid=766, compaction requested=false 2024-11-14T09:13:58,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for a92123abacf02fce4f65c3ba94256e83: 2024-11-14T09:13:58,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:58,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-14T09:13:58,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-14T09:13:58,428 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-14T09:13:58,428 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 868 msec 2024-11-14T09:13:58,431 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 873 msec 2024-11-14T09:13:58,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-14T09:13:58,695 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:13:58,695 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:13:58,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-14T09:13:58,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-14T09:13:58,698 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:13:58,699 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:13:58,699 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:13:58,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on a92123abacf02fce4f65c3ba94256e83 2024-11-14T09:13:58,727 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a92123abacf02fce4f65c3ba94256e83 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-14T09:13:58,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=A 2024-11-14T09:13:58,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:13:58,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=B 2024-11-14T09:13:58,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:13:58,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=C 2024-11-14T09:13:58,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:13:58,740 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/12e846cdd6574f86a2eca6a8aaca5ef2 is 50, key is test_row_0/A:col10/1731575638725/Put/seqid=0 2024-11-14T09:13:58,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741906_1082 (size=14741) 2024-11-14T09:13:58,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:58,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 251 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575698765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:58,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:58,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:58,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:58,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:58,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575698765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:58,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:58,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:58,770 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:58,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-14T09:13:58,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-14T09:13:58,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:58,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:13:58,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:58,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:58,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:58,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:58,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:58,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575698875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:58,878 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:58,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:58,878 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:58,878 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:58,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575698876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:58,879 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:58,879 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:58,879 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:59,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-14T09:13:59,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:59,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:13:59,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:59,007 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:59,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:59,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:59,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-14T09:13:59,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:59,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575699085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:59,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:59,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:59,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:59,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:59,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575699085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:59,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:59,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:59,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:59,145 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=783 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/12e846cdd6574f86a2eca6a8aaca5ef2 2024-11-14T09:13:59,158 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/f613b1d955e24f5fb154ba100a9766d4 is 50, key is test_row_0/B:col10/1731575638725/Put/seqid=0 2024-11-14T09:13:59,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-14T09:13:59,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:59,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:13:59,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:59,162 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:59,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:59,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:59,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741907_1083 (size=12301) 2024-11-14T09:13:59,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-14T09:13:59,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:59,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:13:59,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:59,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:59,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:59,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:59,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-14T09:13:59,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:59,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:59,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575699395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:59,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575699395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:59,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:59,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:59,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:59,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:59,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:59,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:59,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-14T09:13:59,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:59,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:13:59,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:59,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:59,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:59,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:59,580 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=783 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/f613b1d955e24f5fb154ba100a9766d4 2024-11-14T09:13:59,592 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/fb4bb9e885bb433c8ef2d1293812f174 is 50, key is test_row_0/C:col10/1731575638725/Put/seqid=0 2024-11-14T09:13:59,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741908_1084 (size=12301) 2024-11-14T09:13:59,621 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=783 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/fb4bb9e885bb433c8ef2d1293812f174 2024-11-14T09:13:59,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-14T09:13:59,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:59,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:13:59,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:59,625 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:59,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:59,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:13:59,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/12e846cdd6574f86a2eca6a8aaca5ef2 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/12e846cdd6574f86a2eca6a8aaca5ef2 2024-11-14T09:13:59,639 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/12e846cdd6574f86a2eca6a8aaca5ef2, entries=200, sequenceid=783, filesize=14.4 K 2024-11-14T09:13:59,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/f613b1d955e24f5fb154ba100a9766d4 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/f613b1d955e24f5fb154ba100a9766d4 2024-11-14T09:13:59,647 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/f613b1d955e24f5fb154ba100a9766d4, entries=150, sequenceid=783, filesize=12.0 K 2024-11-14T09:13:59,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/fb4bb9e885bb433c8ef2d1293812f174 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/fb4bb9e885bb433c8ef2d1293812f174 2024-11-14T09:13:59,658 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/fb4bb9e885bb433c8ef2d1293812f174, entries=150, sequenceid=783, filesize=12.0 K 2024-11-14T09:13:59,661 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for a92123abacf02fce4f65c3ba94256e83 in 933ms, sequenceid=783, compaction requested=true 2024-11-14T09:13:59,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a92123abacf02fce4f65c3ba94256e83: 2024-11-14T09:13:59,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a92123abacf02fce4f65c3ba94256e83:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:13:59,661 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:13:59,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:13:59,661 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:13:59,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a92123abacf02fce4f65c3ba94256e83:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:13:59,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:13:59,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a92123abacf02fce4f65c3ba94256e83:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:13:59,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:13:59,664 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40059 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:13:59,664 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:13:59,664 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): a92123abacf02fce4f65c3ba94256e83/A is initiating minor compaction (all files) 2024-11-14T09:13:59,664 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): a92123abacf02fce4f65c3ba94256e83/B is initiating minor compaction (all files) 2024-11-14T09:13:59,664 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a92123abacf02fce4f65c3ba94256e83/A in TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:59,664 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a92123abacf02fce4f65c3ba94256e83/B in TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:59,665 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/2a83db7c6d144326a946f31abaedd4fb, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/38285b08250d4fe8a5aa7c6ce77227e9, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/f613b1d955e24f5fb154ba100a9766d4] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp, totalSize=36.7 K 2024-11-14T09:13:59,665 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/96ea282569d94124b9bf0cfa01b17e5b, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/d1ed19fcc97b45829d8cd7bfacee640f, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/12e846cdd6574f86a2eca6a8aaca5ef2] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp, totalSize=39.1 K 2024-11-14T09:13:59,665 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a83db7c6d144326a946f31abaedd4fb, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=747, earliestPutTs=1731575637228 2024-11-14T09:13:59,665 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96ea282569d94124b9bf0cfa01b17e5b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=747, earliestPutTs=1731575637228 2024-11-14T09:13:59,665 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 38285b08250d4fe8a5aa7c6ce77227e9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=766, earliestPutTs=1731575637380 2024-11-14T09:13:59,666 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting f613b1d955e24f5fb154ba100a9766d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=783, earliestPutTs=1731575638077 2024-11-14T09:13:59,666 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1ed19fcc97b45829d8cd7bfacee640f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=766, earliestPutTs=1731575637380 2024-11-14T09:13:59,667 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12e846cdd6574f86a2eca6a8aaca5ef2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=783, earliestPutTs=1731575638077 2024-11-14T09:13:59,676 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a92123abacf02fce4f65c3ba94256e83#B#compaction#72 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:13:59,677 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/e10190d283024a0195e408f393d5937e is 50, key is test_row_0/B:col10/1731575638725/Put/seqid=0 2024-11-14T09:13:59,703 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a92123abacf02fce4f65c3ba94256e83#A#compaction#73 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:13:59,704 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/0fa5eb9b15c94bb789b09a316273579d is 50, key is test_row_0/A:col10/1731575638725/Put/seqid=0 2024-11-14T09:13:59,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741910_1086 (size=13119) 2024-11-14T09:13:59,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741909_1085 (size=13119) 2024-11-14T09:13:59,725 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/e10190d283024a0195e408f393d5937e as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/e10190d283024a0195e408f393d5937e 2024-11-14T09:13:59,733 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a92123abacf02fce4f65c3ba94256e83/B of a92123abacf02fce4f65c3ba94256e83 into e10190d283024a0195e408f393d5937e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:13:59,733 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a92123abacf02fce4f65c3ba94256e83: 2024-11-14T09:13:59,733 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., storeName=a92123abacf02fce4f65c3ba94256e83/B, priority=13, startTime=1731575639661; duration=0sec 2024-11-14T09:13:59,734 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:13:59,734 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a92123abacf02fce4f65c3ba94256e83:B 2024-11-14T09:13:59,734 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:13:59,735 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:13:59,735 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): a92123abacf02fce4f65c3ba94256e83/C is initiating minor compaction (all files) 2024-11-14T09:13:59,735 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a92123abacf02fce4f65c3ba94256e83/C in TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:59,735 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/9798e58624a94992808a8020bd535514, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/9a8d65c63adf45b5bdfac78669c2378a, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/fb4bb9e885bb433c8ef2d1293812f174] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp, totalSize=36.7 K 2024-11-14T09:13:59,735 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 9798e58624a94992808a8020bd535514, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=747, earliestPutTs=1731575637228 2024-11-14T09:13:59,736 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a8d65c63adf45b5bdfac78669c2378a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=766, earliestPutTs=1731575637380 2024-11-14T09:13:59,737 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting fb4bb9e885bb433c8ef2d1293812f174, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=783, earliestPutTs=1731575638077 2024-11-14T09:13:59,749 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a92123abacf02fce4f65c3ba94256e83#C#compaction#74 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:13:59,750 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/6145abc98fa94e4a82bf9728ddb68e15 is 50, key is test_row_0/C:col10/1731575638725/Put/seqid=0 2024-11-14T09:13:59,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741911_1087 (size=13119) 2024-11-14T09:13:59,775 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/6145abc98fa94e4a82bf9728ddb68e15 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/6145abc98fa94e4a82bf9728ddb68e15 2024-11-14T09:13:59,783 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a92123abacf02fce4f65c3ba94256e83/C of a92123abacf02fce4f65c3ba94256e83 into 6145abc98fa94e4a82bf9728ddb68e15(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:13:59,783 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a92123abacf02fce4f65c3ba94256e83: 2024-11-14T09:13:59,783 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., storeName=a92123abacf02fce4f65c3ba94256e83/C, priority=13, startTime=1731575639662; duration=0sec 2024-11-14T09:13:59,783 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:13:59,783 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a92123abacf02fce4f65c3ba94256e83:C 2024-11-14T09:13:59,788 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-14T09:13:59,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:13:59,789 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing a92123abacf02fce4f65c3ba94256e83 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-14T09:13:59,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=A 2024-11-14T09:13:59,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:13:59,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=B 2024-11-14T09:13:59,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:13:59,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=C 2024-11-14T09:13:59,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:13:59,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/0ada844e0ac24df2a410bcaaee5e44d0 is 50, key is test_row_0/A:col10/1731575638758/Put/seqid=0 2024-11-14T09:13:59,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741912_1088 (size=12301) 2024-11-14T09:13:59,815 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=804 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/0ada844e0ac24df2a410bcaaee5e44d0 2024-11-14T09:13:59,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/d18da34ee95d44758e3fca3dafd2a9b7 is 50, key is test_row_0/B:col10/1731575638758/Put/seqid=0 2024-11-14T09:13:59,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-14T09:13:59,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741913_1089 (size=12301) 2024-11-14T09:13:59,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:13:59,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on a92123abacf02fce4f65c3ba94256e83 2024-11-14T09:13:59,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:59,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 260 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575699931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:59,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:13:59,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:59,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:59,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575699932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:13:59,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:13:59,939 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:59,939 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:13:59,939 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:00,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:00,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 261 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575700045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:00,046 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:00,046 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:00,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:00,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:00,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575700045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:00,047 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:00,047 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:00,047 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:00,121 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/0fa5eb9b15c94bb789b09a316273579d as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/0fa5eb9b15c94bb789b09a316273579d 2024-11-14T09:14:00,128 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a92123abacf02fce4f65c3ba94256e83/A of a92123abacf02fce4f65c3ba94256e83 into 0fa5eb9b15c94bb789b09a316273579d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:14:00,128 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a92123abacf02fce4f65c3ba94256e83: 2024-11-14T09:14:00,128 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., storeName=a92123abacf02fce4f65c3ba94256e83/A, priority=13, startTime=1731575639661; duration=0sec 2024-11-14T09:14:00,128 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:00,128 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a92123abacf02fce4f65c3ba94256e83:A 2024-11-14T09:14:00,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:00,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 262 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575700255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:00,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:00,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:00,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575700255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:00,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:00,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:00,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:00,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:00,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:00,272 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=804 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/d18da34ee95d44758e3fca3dafd2a9b7 2024-11-14T09:14:00,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/7057ca4cc01d4276b9f7ab1aedfdcb07 is 50, key is test_row_0/C:col10/1731575638758/Put/seqid=0 2024-11-14T09:14:00,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741914_1090 (size=12301) 2024-11-14T09:14:00,294 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=804 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/7057ca4cc01d4276b9f7ab1aedfdcb07 2024-11-14T09:14:00,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/0ada844e0ac24df2a410bcaaee5e44d0 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/0ada844e0ac24df2a410bcaaee5e44d0 2024-11-14T09:14:00,307 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/0ada844e0ac24df2a410bcaaee5e44d0, entries=150, sequenceid=804, filesize=12.0 K 2024-11-14T09:14:00,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/d18da34ee95d44758e3fca3dafd2a9b7 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/d18da34ee95d44758e3fca3dafd2a9b7 2024-11-14T09:14:00,314 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/d18da34ee95d44758e3fca3dafd2a9b7, entries=150, sequenceid=804, filesize=12.0 K 2024-11-14T09:14:00,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/7057ca4cc01d4276b9f7ab1aedfdcb07 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/7057ca4cc01d4276b9f7ab1aedfdcb07 2024-11-14T09:14:00,323 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/7057ca4cc01d4276b9f7ab1aedfdcb07, entries=150, sequenceid=804, filesize=12.0 K 2024-11-14T09:14:00,324 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for a92123abacf02fce4f65c3ba94256e83 in 535ms, sequenceid=804, compaction requested=false 2024-11-14T09:14:00,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for a92123abacf02fce4f65c3ba94256e83: 2024-11-14T09:14:00,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:00,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-14T09:14:00,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-14T09:14:00,329 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-14T09:14:00,329 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6280 sec 2024-11-14T09:14:00,331 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.6350 sec 2024-11-14T09:14:00,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on a92123abacf02fce4f65c3ba94256e83 2024-11-14T09:14:00,577 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a92123abacf02fce4f65c3ba94256e83 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-14T09:14:00,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=A 2024-11-14T09:14:00,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:00,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=B 2024-11-14T09:14:00,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:00,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=C 2024-11-14T09:14:00,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:00,584 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/a9d1625c16164fd69c396896465cfd64 is 50, key is test_row_0/A:col10/1731575640577/Put/seqid=0 2024-11-14T09:14:00,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741915_1091 (size=14741) 2024-11-14T09:14:00,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:00,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 271 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575700610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:00,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:00,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:00,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:00,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:00,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575700614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:00,619 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:00,619 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:00,619 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:00,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:00,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 272 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575700725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:00,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:00,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:00,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:00,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575700725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:00,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:00,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:00,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:00,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:00,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-14T09:14:00,846 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:14:00,846 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:14:00,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-14T09:14:00,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T09:14:00,848 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:14:00,849 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:14:00,849 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:14:00,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:00,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 273 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575700935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:00,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:00,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:00,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:00,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:00,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575700935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:00,938 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:00,938 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:00,938 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:00,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T09:14:00,990 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=820 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/a9d1625c16164fd69c396896465cfd64 2024-11-14T09:14:01,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-14T09:14:01,001 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/f26daec2a3ba4645a31e89ba7df5a719 is 50, key is test_row_0/B:col10/1731575640577/Put/seqid=0 2024-11-14T09:14:01,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:01,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:14:01,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:01,002 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:01,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:01,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:01,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741916_1092 (size=12301) 2024-11-14T09:14:01,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-14T09:14:01,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:01,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:14:01,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:01,155 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:01,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:01,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:01,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T09:14:01,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:01,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 274 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575701245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:01,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:01,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 256 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575701245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:01,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:01,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:01,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:01,250 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:01,250 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:01,250 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:01,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-14T09:14:01,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:01,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:14:01,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:01,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:01,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:01,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:01,411 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=820 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/f26daec2a3ba4645a31e89ba7df5a719 2024-11-14T09:14:01,423 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/69edb8844b8044079dca1ff888d4aba6 is 50, key is test_row_0/C:col10/1731575640577/Put/seqid=0 2024-11-14T09:14:01,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741917_1093 (size=12301) 2024-11-14T09:14:01,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-14T09:14:01,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:01,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:14:01,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:01,464 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:01,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:01,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:01,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T09:14:01,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-14T09:14:01,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:01,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:14:01,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:01,618 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:01,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:01,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:01,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:01,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 275 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575701755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:01,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:01,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:01,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:01,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 257 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575701755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:01,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:01,757 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:01,757 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:01,758 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:01,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-14T09:14:01,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:01,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:14:01,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:01,771 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:01,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:01,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:01,832 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=820 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/69edb8844b8044079dca1ff888d4aba6 2024-11-14T09:14:01,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:01,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49966 deadline: 1731575701835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:01,839 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_0' in a92123abacf02fce4f65c3ba94256e83 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:14:01,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/a9d1625c16164fd69c396896465cfd64 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/a9d1625c16164fd69c396896465cfd64 2024-11-14T09:14:01,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:01,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:01,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:01,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:01,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49980 deadline: 1731575701846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:01,848 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/a9d1625c16164fd69c396896465cfd64, entries=200, sequenceid=820, filesize=14.4 K 2024-11-14T09:14:01,850 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_1' in a92123abacf02fce4f65c3ba94256e83 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:14:01,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:01,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:01,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:01,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/f26daec2a3ba4645a31e89ba7df5a719 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/f26daec2a3ba4645a31e89ba7df5a719 2024-11-14T09:14:01,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:01,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49996 deadline: 1731575701855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:01,857 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_1' in a92123abacf02fce4f65c3ba94256e83 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:14:01,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:01,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:01,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:01,862 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/f26daec2a3ba4645a31e89ba7df5a719, entries=150, sequenceid=820, filesize=12.0 K 2024-11-14T09:14:01,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/69edb8844b8044079dca1ff888d4aba6 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/69edb8844b8044079dca1ff888d4aba6 2024-11-14T09:14:01,874 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/69edb8844b8044079dca1ff888d4aba6, entries=150, sequenceid=820, filesize=12.0 K 2024-11-14T09:14:01,875 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for a92123abacf02fce4f65c3ba94256e83 in 1298ms, sequenceid=820, compaction requested=true 2024-11-14T09:14:01,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a92123abacf02fce4f65c3ba94256e83: 2024-11-14T09:14:01,876 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:14:01,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a92123abacf02fce4f65c3ba94256e83:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:14:01,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:01,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a92123abacf02fce4f65c3ba94256e83:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:14:01,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:01,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a92123abacf02fce4f65c3ba94256e83:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:14:01,877 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:14:01,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:14:01,878 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40161 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:14:01,878 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): a92123abacf02fce4f65c3ba94256e83/A is initiating minor compaction (all files) 2024-11-14T09:14:01,878 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a92123abacf02fce4f65c3ba94256e83/A in TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:01,878 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/0fa5eb9b15c94bb789b09a316273579d, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/0ada844e0ac24df2a410bcaaee5e44d0, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/a9d1625c16164fd69c396896465cfd64] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp, totalSize=39.2 K 2024-11-14T09:14:01,879 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:14:01,879 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): a92123abacf02fce4f65c3ba94256e83/B is initiating minor compaction (all files) 2024-11-14T09:14:01,880 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a92123abacf02fce4f65c3ba94256e83/B in TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:01,880 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/e10190d283024a0195e408f393d5937e, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/d18da34ee95d44758e3fca3dafd2a9b7, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/f26daec2a3ba4645a31e89ba7df5a719] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp, totalSize=36.8 K 2024-11-14T09:14:01,880 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0fa5eb9b15c94bb789b09a316273579d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=783, earliestPutTs=1731575638077 2024-11-14T09:14:01,881 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting e10190d283024a0195e408f393d5937e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=783, earliestPutTs=1731575638077 2024-11-14T09:14:01,881 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting d18da34ee95d44758e3fca3dafd2a9b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=804, earliestPutTs=1731575638758 2024-11-14T09:14:01,881 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ada844e0ac24df2a410bcaaee5e44d0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=804, earliestPutTs=1731575638758 2024-11-14T09:14:01,882 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9d1625c16164fd69c396896465cfd64, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=820, earliestPutTs=1731575639927 2024-11-14T09:14:01,882 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting f26daec2a3ba4645a31e89ba7df5a719, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=820, earliestPutTs=1731575639929 2024-11-14T09:14:01,901 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a92123abacf02fce4f65c3ba94256e83#A#compaction#81 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:14:01,902 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/10247dd911384f3c815184f3272ed175 is 50, key is test_row_0/A:col10/1731575640577/Put/seqid=0 2024-11-14T09:14:01,904 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a92123abacf02fce4f65c3ba94256e83#B#compaction#82 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:14:01,905 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/2debd4527a894a8e98ff11108a7f8066 is 50, key is test_row_0/B:col10/1731575640577/Put/seqid=0 2024-11-14T09:14:01,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741918_1094 (size=13221) 2024-11-14T09:14:01,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741919_1095 (size=13221) 2024-11-14T09:14:01,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-14T09:14:01,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:01,925 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing a92123abacf02fce4f65c3ba94256e83 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-14T09:14:01,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=A 2024-11-14T09:14:01,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:01,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=B 2024-11-14T09:14:01,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:01,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=C 2024-11-14T09:14:01,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:01,933 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/10247dd911384f3c815184f3272ed175 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/10247dd911384f3c815184f3272ed175 2024-11-14T09:14:01,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/8d353be469204ab090c0315eb9c94dd6 is 50, key is test_row_0/A:col10/1731575640612/Put/seqid=0 2024-11-14T09:14:01,943 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a92123abacf02fce4f65c3ba94256e83/A of a92123abacf02fce4f65c3ba94256e83 into 10247dd911384f3c815184f3272ed175(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:14:01,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741920_1096 (size=12301) 2024-11-14T09:14:01,943 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a92123abacf02fce4f65c3ba94256e83: 2024-11-14T09:14:01,943 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., storeName=a92123abacf02fce4f65c3ba94256e83/A, priority=13, startTime=1731575641876; duration=0sec 2024-11-14T09:14:01,944 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:14:01,945 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a92123abacf02fce4f65c3ba94256e83:A 2024-11-14T09:14:01,945 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:14:01,947 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:14:01,947 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): a92123abacf02fce4f65c3ba94256e83/C is initiating minor compaction (all files) 2024-11-14T09:14:01,947 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a92123abacf02fce4f65c3ba94256e83/C in TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:01,947 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/6145abc98fa94e4a82bf9728ddb68e15, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/7057ca4cc01d4276b9f7ab1aedfdcb07, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/69edb8844b8044079dca1ff888d4aba6] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp, totalSize=36.8 K 2024-11-14T09:14:01,948 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6145abc98fa94e4a82bf9728ddb68e15, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=783, earliestPutTs=1731575638077 2024-11-14T09:14:01,948 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7057ca4cc01d4276b9f7ab1aedfdcb07, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=804, earliestPutTs=1731575638758 2024-11-14T09:14:01,948 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69edb8844b8044079dca1ff888d4aba6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=820, earliestPutTs=1731575639929 2024-11-14T09:14:01,960 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a92123abacf02fce4f65c3ba94256e83#C#compaction#84 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:14:01,961 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/00fc75d98be34de59fe98a9173cd9780 is 50, key is test_row_0/C:col10/1731575640577/Put/seqid=0 2024-11-14T09:14:01,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741921_1097 (size=13221) 2024-11-14T09:14:01,980 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/00fc75d98be34de59fe98a9173cd9780 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/00fc75d98be34de59fe98a9173cd9780 2024-11-14T09:14:01,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T09:14:01,988 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a92123abacf02fce4f65c3ba94256e83/C of a92123abacf02fce4f65c3ba94256e83 into 00fc75d98be34de59fe98a9173cd9780(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:14:01,988 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a92123abacf02fce4f65c3ba94256e83: 2024-11-14T09:14:01,988 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., storeName=a92123abacf02fce4f65c3ba94256e83/C, priority=13, startTime=1731575641877; duration=0sec 2024-11-14T09:14:01,988 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:01,988 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a92123abacf02fce4f65c3ba94256e83:C 2024-11-14T09:14:02,330 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/2debd4527a894a8e98ff11108a7f8066 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/2debd4527a894a8e98ff11108a7f8066 2024-11-14T09:14:02,336 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a92123abacf02fce4f65c3ba94256e83/B of a92123abacf02fce4f65c3ba94256e83 into 2debd4527a894a8e98ff11108a7f8066(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:14:02,336 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a92123abacf02fce4f65c3ba94256e83: 2024-11-14T09:14:02,336 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., storeName=a92123abacf02fce4f65c3ba94256e83/B, priority=13, startTime=1731575641877; duration=0sec 2024-11-14T09:14:02,336 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:02,336 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a92123abacf02fce4f65c3ba94256e83:B 2024-11-14T09:14:02,343 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=837 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/8d353be469204ab090c0315eb9c94dd6 2024-11-14T09:14:02,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/aeb2ce283ea34d63a8c1f85238832986 is 50, key is test_row_0/B:col10/1731575640612/Put/seqid=0 2024-11-14T09:14:02,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741922_1098 (size=12301) 2024-11-14T09:14:02,762 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=837 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/aeb2ce283ea34d63a8c1f85238832986 2024-11-14T09:14:02,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on a92123abacf02fce4f65c3ba94256e83 2024-11-14T09:14:02,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:14:02,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/ce3cf758e9f840899c0394c939f03495 is 50, key is test_row_0/C:col10/1731575640612/Put/seqid=0 2024-11-14T09:14:02,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741923_1099 (size=12301) 2024-11-14T09:14:02,795 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=837 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/ce3cf758e9f840899c0394c939f03495 2024-11-14T09:14:02,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:02,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 263 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575702797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:02,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:02,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:02,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:02,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/8d353be469204ab090c0315eb9c94dd6 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/8d353be469204ab090c0315eb9c94dd6 2024-11-14T09:14:02,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:02,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 283 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575702801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:02,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:02,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:02,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:02,809 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/8d353be469204ab090c0315eb9c94dd6, entries=150, sequenceid=837, filesize=12.0 K 2024-11-14T09:14:02,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/aeb2ce283ea34d63a8c1f85238832986 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/aeb2ce283ea34d63a8c1f85238832986 2024-11-14T09:14:02,818 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/aeb2ce283ea34d63a8c1f85238832986, entries=150, sequenceid=837, filesize=12.0 K 2024-11-14T09:14:02,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/ce3cf758e9f840899c0394c939f03495 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/ce3cf758e9f840899c0394c939f03495 2024-11-14T09:14:02,829 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/ce3cf758e9f840899c0394c939f03495, entries=150, sequenceid=837, filesize=12.0 K 2024-11-14T09:14:02,830 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for a92123abacf02fce4f65c3ba94256e83 in 905ms, sequenceid=837, compaction requested=false 2024-11-14T09:14:02,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for a92123abacf02fce4f65c3ba94256e83: 2024-11-14T09:14:02,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:02,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-14T09:14:02,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-14T09:14:02,834 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-14T09:14:02,834 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9830 sec 2024-11-14T09:14:02,837 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.9900 sec 2024-11-14T09:14:02,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on a92123abacf02fce4f65c3ba94256e83 2024-11-14T09:14:02,909 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a92123abacf02fce4f65c3ba94256e83 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-14T09:14:02,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=A 2024-11-14T09:14:02,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:02,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=B 2024-11-14T09:14:02,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:02,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=C 2024-11-14T09:14:02,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:02,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/0a6cbc5477cb44ecb7c7b912a6e03f4f is 50, key is test_row_0/A:col10/1731575642796/Put/seqid=0 2024-11-14T09:14:02,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741924_1100 (size=14741) 2024-11-14T09:14:02,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:02,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 272 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575702951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:02,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:02,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:02,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:02,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:02,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 290 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575702951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:02,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:02,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:02,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:03,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T09:14:03,005 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:14:03,006 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:14:03,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-14T09:14:03,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T09:14:03,008 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:14:03,009 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:14:03,009 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:14:03,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:03,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 273 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575703054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:03,057 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:03,058 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:03,058 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:03,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:03,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 291 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575703065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:03,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:03,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:03,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:03,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T09:14:03,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-14T09:14:03,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:03,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:14:03,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:03,162 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:03,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:03,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:03,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:03,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 274 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575703265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:03,267 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:03,267 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:03,267 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:03,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:03,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 292 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575703275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:03,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:03,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:03,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:03,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-14T09:14:03,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:03,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:14:03,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:03,316 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:03,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:03,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:03,324 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=856 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/0a6cbc5477cb44ecb7c7b912a6e03f4f 2024-11-14T09:14:03,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T09:14:03,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/d8623bdb4fd141dc98d37c0930f077dd is 50, key is test_row_0/B:col10/1731575642796/Put/seqid=0 2024-11-14T09:14:03,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741925_1101 (size=12301) 2024-11-14T09:14:03,373 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=856 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/d8623bdb4fd141dc98d37c0930f077dd 2024-11-14T09:14:03,387 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/07d3f5dc69cc4c61b17ccd3f22c85625 is 50, key is test_row_0/C:col10/1731575642796/Put/seqid=0 2024-11-14T09:14:03,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741926_1102 (size=12301) 2024-11-14T09:14:03,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-14T09:14:03,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:03,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:14:03,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:03,470 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:03,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:03,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:03,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:03,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 275 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575703574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:03,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:03,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:03,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:03,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:03,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 293 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575703585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:03,588 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:03,588 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:03,588 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:03,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-14T09:14:03,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:03,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:14:03,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:03,628 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:03,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:03,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:03,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T09:14:03,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-14T09:14:03,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:03,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:14:03,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:03,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:03,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:03,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:03,793 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=856 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/07d3f5dc69cc4c61b17ccd3f22c85625 2024-11-14T09:14:03,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/0a6cbc5477cb44ecb7c7b912a6e03f4f as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/0a6cbc5477cb44ecb7c7b912a6e03f4f 2024-11-14T09:14:03,812 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/0a6cbc5477cb44ecb7c7b912a6e03f4f, entries=200, sequenceid=856, filesize=14.4 K 2024-11-14T09:14:03,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/d8623bdb4fd141dc98d37c0930f077dd as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/d8623bdb4fd141dc98d37c0930f077dd 2024-11-14T09:14:03,823 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/d8623bdb4fd141dc98d37c0930f077dd, entries=150, sequenceid=856, filesize=12.0 K 2024-11-14T09:14:03,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/07d3f5dc69cc4c61b17ccd3f22c85625 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/07d3f5dc69cc4c61b17ccd3f22c85625 2024-11-14T09:14:03,835 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/07d3f5dc69cc4c61b17ccd3f22c85625, entries=150, sequenceid=856, filesize=12.0 K 2024-11-14T09:14:03,836 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for a92123abacf02fce4f65c3ba94256e83 in 927ms, sequenceid=856, compaction requested=true 2024-11-14T09:14:03,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a92123abacf02fce4f65c3ba94256e83: 2024-11-14T09:14:03,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a92123abacf02fce4f65c3ba94256e83:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:14:03,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:03,837 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:14:03,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a92123abacf02fce4f65c3ba94256e83:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:14:03,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:03,837 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:14:03,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a92123abacf02fce4f65c3ba94256e83:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:14:03,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:14:03,839 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:14:03,839 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40263 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:14:03,839 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): a92123abacf02fce4f65c3ba94256e83/A is initiating minor compaction (all files) 2024-11-14T09:14:03,839 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): a92123abacf02fce4f65c3ba94256e83/B is initiating minor compaction (all files) 2024-11-14T09:14:03,839 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a92123abacf02fce4f65c3ba94256e83/A in TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:03,839 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a92123abacf02fce4f65c3ba94256e83/B in TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:03,839 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/2debd4527a894a8e98ff11108a7f8066, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/aeb2ce283ea34d63a8c1f85238832986, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/d8623bdb4fd141dc98d37c0930f077dd] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp, totalSize=36.9 K 2024-11-14T09:14:03,839 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/10247dd911384f3c815184f3272ed175, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/8d353be469204ab090c0315eb9c94dd6, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/0a6cbc5477cb44ecb7c7b912a6e03f4f] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp, totalSize=39.3 K 2024-11-14T09:14:03,840 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 2debd4527a894a8e98ff11108a7f8066, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=820, earliestPutTs=1731575639929 2024-11-14T09:14:03,840 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10247dd911384f3c815184f3272ed175, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=820, earliestPutTs=1731575639929 2024-11-14T09:14:03,841 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting aeb2ce283ea34d63a8c1f85238832986, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=837, earliestPutTs=1731575640591 2024-11-14T09:14:03,841 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting d8623bdb4fd141dc98d37c0930f077dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=856, earliestPutTs=1731575642793 2024-11-14T09:14:03,841 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d353be469204ab090c0315eb9c94dd6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=837, earliestPutTs=1731575640591 2024-11-14T09:14:03,842 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a6cbc5477cb44ecb7c7b912a6e03f4f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=856, earliestPutTs=1731575642781 2024-11-14T09:14:03,861 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a92123abacf02fce4f65c3ba94256e83#B#compaction#90 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:14:03,862 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/4adfc20ad13443079b8e9ad2f5ddcc09 is 50, key is test_row_0/B:col10/1731575642796/Put/seqid=0 2024-11-14T09:14:03,866 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a92123abacf02fce4f65c3ba94256e83#A#compaction#91 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:14:03,867 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/99819218d62a44e5855eae38eb83d3aa is 50, key is test_row_0/A:col10/1731575642796/Put/seqid=0 2024-11-14T09:14:03,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741927_1103 (size=13323) 2024-11-14T09:14:03,887 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/4adfc20ad13443079b8e9ad2f5ddcc09 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/4adfc20ad13443079b8e9ad2f5ddcc09 2024-11-14T09:14:03,896 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a92123abacf02fce4f65c3ba94256e83/B of a92123abacf02fce4f65c3ba94256e83 into 4adfc20ad13443079b8e9ad2f5ddcc09(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:14:03,896 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a92123abacf02fce4f65c3ba94256e83: 2024-11-14T09:14:03,896 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., storeName=a92123abacf02fce4f65c3ba94256e83/B, priority=13, startTime=1731575643837; duration=0sec 2024-11-14T09:14:03,897 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:14:03,897 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a92123abacf02fce4f65c3ba94256e83:B 2024-11-14T09:14:03,897 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:14:03,898 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:14:03,898 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): a92123abacf02fce4f65c3ba94256e83/C is initiating minor compaction (all files) 2024-11-14T09:14:03,898 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a92123abacf02fce4f65c3ba94256e83/C in TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:03,899 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/00fc75d98be34de59fe98a9173cd9780, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/ce3cf758e9f840899c0394c939f03495, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/07d3f5dc69cc4c61b17ccd3f22c85625] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp, totalSize=36.9 K 2024-11-14T09:14:03,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741928_1104 (size=13323) 2024-11-14T09:14:03,899 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 00fc75d98be34de59fe98a9173cd9780, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=820, earliestPutTs=1731575639929 2024-11-14T09:14:03,900 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting ce3cf758e9f840899c0394c939f03495, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=837, earliestPutTs=1731575640591 2024-11-14T09:14:03,907 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 07d3f5dc69cc4c61b17ccd3f22c85625, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=856, earliestPutTs=1731575642793 2024-11-14T09:14:03,912 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/99819218d62a44e5855eae38eb83d3aa as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/99819218d62a44e5855eae38eb83d3aa 2024-11-14T09:14:03,920 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a92123abacf02fce4f65c3ba94256e83/A of a92123abacf02fce4f65c3ba94256e83 into 99819218d62a44e5855eae38eb83d3aa(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:14:03,920 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a92123abacf02fce4f65c3ba94256e83: 2024-11-14T09:14:03,920 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., storeName=a92123abacf02fce4f65c3ba94256e83/A, priority=13, startTime=1731575643837; duration=0sec 2024-11-14T09:14:03,920 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:03,920 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a92123abacf02fce4f65c3ba94256e83:A 2024-11-14T09:14:03,926 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a92123abacf02fce4f65c3ba94256e83#C#compaction#92 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:14:03,927 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/813a8032b4da43df9a70af6c6833cbbf is 50, key is test_row_0/C:col10/1731575642796/Put/seqid=0 2024-11-14T09:14:03,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-14T09:14:03,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:03,938 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing a92123abacf02fce4f65c3ba94256e83 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-14T09:14:03,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=A 2024-11-14T09:14:03,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:03,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=B 2024-11-14T09:14:03,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:03,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=C 2024-11-14T09:14:03,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:03,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/141a797724fb4576b5a126957eec4a1b is 50, key is test_row_0/A:col10/1731575642942/Put/seqid=0 2024-11-14T09:14:03,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741929_1105 (size=13323) 2024-11-14T09:14:04,005 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/813a8032b4da43df9a70af6c6833cbbf as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/813a8032b4da43df9a70af6c6833cbbf 2024-11-14T09:14:04,017 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a92123abacf02fce4f65c3ba94256e83/C of a92123abacf02fce4f65c3ba94256e83 into 813a8032b4da43df9a70af6c6833cbbf(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:14:04,017 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a92123abacf02fce4f65c3ba94256e83: 2024-11-14T09:14:04,018 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., storeName=a92123abacf02fce4f65c3ba94256e83/C, priority=13, startTime=1731575643837; duration=0sec 2024-11-14T09:14:04,019 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:04,019 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a92123abacf02fce4f65c3ba94256e83:C 2024-11-14T09:14:04,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741930_1106 (size=12301) 2024-11-14T09:14:04,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on a92123abacf02fce4f65c3ba94256e83 2024-11-14T09:14:04,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:14:04,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T09:14:04,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:04,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 298 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575704157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:04,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:04,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:04,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:04,166 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:04,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 285 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575704162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:04,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:04,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:04,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:04,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:04,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 299 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575704275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:04,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:04,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 286 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575704275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:04,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:04,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:04,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:04,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:04,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:04,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:04,436 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=874 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/141a797724fb4576b5a126957eec4a1b 2024-11-14T09:14:04,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/16d44bf4b50d4f4b84e5f5e31a341c6b is 50, key is test_row_0/B:col10/1731575642942/Put/seqid=0 2024-11-14T09:14:04,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741931_1107 (size=12301) 2024-11-14T09:14:04,471 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=874 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/16d44bf4b50d4f4b84e5f5e31a341c6b 2024-11-14T09:14:04,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/1fa947d9d8fe44baac4a79c508efd4f6 is 50, key is test_row_0/C:col10/1731575642942/Put/seqid=0 2024-11-14T09:14:04,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:04,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 287 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575704485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:04,487 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:04,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:04,487 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:04,487 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:04,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 300 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575704485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:04,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:04,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:04,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:04,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741932_1108 (size=12301) 2024-11-14T09:14:04,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:04,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 288 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575704794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:04,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:04,797 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:04,797 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:04,797 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:04,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 301 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575704794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:04,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:04,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:04,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:04,893 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=874 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/1fa947d9d8fe44baac4a79c508efd4f6 2024-11-14T09:14:04,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/141a797724fb4576b5a126957eec4a1b as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/141a797724fb4576b5a126957eec4a1b 2024-11-14T09:14:04,907 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/141a797724fb4576b5a126957eec4a1b, entries=150, sequenceid=874, filesize=12.0 K 2024-11-14T09:14:04,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/16d44bf4b50d4f4b84e5f5e31a341c6b as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/16d44bf4b50d4f4b84e5f5e31a341c6b 2024-11-14T09:14:04,916 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/16d44bf4b50d4f4b84e5f5e31a341c6b, entries=150, sequenceid=874, filesize=12.0 K 2024-11-14T09:14:04,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/1fa947d9d8fe44baac4a79c508efd4f6 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/1fa947d9d8fe44baac4a79c508efd4f6 2024-11-14T09:14:04,928 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/1fa947d9d8fe44baac4a79c508efd4f6, entries=150, sequenceid=874, filesize=12.0 K 2024-11-14T09:14:04,931 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for a92123abacf02fce4f65c3ba94256e83 in 993ms, sequenceid=874, compaction requested=false 2024-11-14T09:14:04,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for a92123abacf02fce4f65c3ba94256e83: 2024-11-14T09:14:04,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:04,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-14T09:14:04,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-14T09:14:04,936 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-14T09:14:04,936 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9240 sec 2024-11-14T09:14:04,940 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.9310 sec 2024-11-14T09:14:05,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T09:14:05,156 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:14:05,156 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:14:05,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-14T09:14:05,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-14T09:14:05,158 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:14:05,159 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:14:05,159 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:14:05,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-14T09:14:05,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on a92123abacf02fce4f65c3ba94256e83 2024-11-14T09:14:05,307 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a92123abacf02fce4f65c3ba94256e83 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-14T09:14:05,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=A 2024-11-14T09:14:05,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:05,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=B 2024-11-14T09:14:05,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:05,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=C 2024-11-14T09:14:05,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:05,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-14T09:14:05,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:05,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:14:05,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:05,313 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:05,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:05,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/459fbb7015be49a49f2ebc9a1303c34c is 50, key is test_row_0/A:col10/1731575644128/Put/seqid=0 2024-11-14T09:14:05,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:05,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741933_1109 (size=12301) 2024-11-14T09:14:05,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:05,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 295 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575705331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:05,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:05,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:05,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:05,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:05,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 309 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575705332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:05,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:05,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:05,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:05,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:05,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 296 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575705435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:05,436 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:05,436 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:05,436 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:05,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:05,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 310 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575705445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:05,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:05,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:05,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:05,466 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-14T09:14:05,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:05,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:14:05,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:05,467 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:05,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:05,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:05,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-14T09:14:05,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-14T09:14:05,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:05,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:14:05,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:05,620 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:05,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:05,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:05,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:05,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 297 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575705645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:05,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:05,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:05,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:05,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:05,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 311 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575705655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:05,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:05,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:05,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:05,723 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=893 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/459fbb7015be49a49f2ebc9a1303c34c 2024-11-14T09:14:05,732 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/584e999ed58c4fb7b78fa02548daa987 is 50, key is test_row_0/B:col10/1731575644128/Put/seqid=0 2024-11-14T09:14:05,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741934_1110 (size=12301) 2024-11-14T09:14:05,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-14T09:14:05,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:05,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:14:05,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:05,774 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:05,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:05,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:05,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-14T09:14:05,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-14T09:14:05,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:05,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:14:05,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:05,927 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:05,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:05,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:05,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:05,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 298 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575705955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:05,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:05,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:05,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:05,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:05,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 312 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575705965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:05,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:05,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:05,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:06,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-14T09:14:06,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:06,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:14:06,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:06,080 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:06,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:06,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:06,137 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=893 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/584e999ed58c4fb7b78fa02548daa987 2024-11-14T09:14:06,147 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/207cf17213694580878b44c1cbbd8edb is 50, key is test_row_0/C:col10/1731575644128/Put/seqid=0 2024-11-14T09:14:06,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741935_1111 (size=12301) 2024-11-14T09:14:06,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-14T09:14:06,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:06,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:14:06,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:06,234 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:06,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:06,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:06,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-14T09:14:06,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-14T09:14:06,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:06,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:14:06,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:06,388 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:06,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:06,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:06,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:06,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 299 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575706465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:06,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:06,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:06,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:06,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:06,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 313 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575706475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:06,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:06,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:06,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:06,542 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-14T09:14:06,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:06,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:14:06,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:06,542 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:06,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:06,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:06,553 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=893 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/207cf17213694580878b44c1cbbd8edb 2024-11-14T09:14:06,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/459fbb7015be49a49f2ebc9a1303c34c as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/459fbb7015be49a49f2ebc9a1303c34c 2024-11-14T09:14:06,566 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/459fbb7015be49a49f2ebc9a1303c34c, entries=150, sequenceid=893, filesize=12.0 K 2024-11-14T09:14:06,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/584e999ed58c4fb7b78fa02548daa987 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/584e999ed58c4fb7b78fa02548daa987 2024-11-14T09:14:06,572 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/584e999ed58c4fb7b78fa02548daa987, entries=150, sequenceid=893, filesize=12.0 K 2024-11-14T09:14:06,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/207cf17213694580878b44c1cbbd8edb as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/207cf17213694580878b44c1cbbd8edb 2024-11-14T09:14:06,581 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/207cf17213694580878b44c1cbbd8edb, entries=150, sequenceid=893, filesize=12.0 K 2024-11-14T09:14:06,582 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for a92123abacf02fce4f65c3ba94256e83 in 1275ms, sequenceid=893, compaction requested=true 2024-11-14T09:14:06,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a92123abacf02fce4f65c3ba94256e83: 2024-11-14T09:14:06,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a92123abacf02fce4f65c3ba94256e83:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:14:06,582 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:14:06,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:06,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a92123abacf02fce4f65c3ba94256e83:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:14:06,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:06,583 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:14:06,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a92123abacf02fce4f65c3ba94256e83:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:14:06,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:14:06,584 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:14:06,584 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): a92123abacf02fce4f65c3ba94256e83/A is initiating minor compaction (all files) 2024-11-14T09:14:06,584 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a92123abacf02fce4f65c3ba94256e83/A in TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:06,584 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/99819218d62a44e5855eae38eb83d3aa, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/141a797724fb4576b5a126957eec4a1b, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/459fbb7015be49a49f2ebc9a1303c34c] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp, totalSize=37.0 K 2024-11-14T09:14:06,584 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:14:06,584 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): a92123abacf02fce4f65c3ba94256e83/B is initiating minor compaction (all files) 2024-11-14T09:14:06,584 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a92123abacf02fce4f65c3ba94256e83/B in TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:06,585 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/4adfc20ad13443079b8e9ad2f5ddcc09, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/16d44bf4b50d4f4b84e5f5e31a341c6b, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/584e999ed58c4fb7b78fa02548daa987] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp, totalSize=37.0 K 2024-11-14T09:14:06,585 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99819218d62a44e5855eae38eb83d3aa, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=856, earliestPutTs=1731575642793 2024-11-14T09:14:06,585 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 4adfc20ad13443079b8e9ad2f5ddcc09, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=856, earliestPutTs=1731575642793 2024-11-14T09:14:06,585 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 141a797724fb4576b5a126957eec4a1b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=874, earliestPutTs=1731575642942 2024-11-14T09:14:06,585 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 16d44bf4b50d4f4b84e5f5e31a341c6b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=874, earliestPutTs=1731575642942 2024-11-14T09:14:06,586 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 459fbb7015be49a49f2ebc9a1303c34c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=893, earliestPutTs=1731575644128 2024-11-14T09:14:06,586 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 584e999ed58c4fb7b78fa02548daa987, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=893, earliestPutTs=1731575644128 2024-11-14T09:14:06,595 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a92123abacf02fce4f65c3ba94256e83#B#compaction#100 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:14:06,596 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a92123abacf02fce4f65c3ba94256e83#A#compaction#99 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:14:06,596 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/b725a107ce9a4562b8637d60faf2628c is 50, key is test_row_0/A:col10/1731575644128/Put/seqid=0 2024-11-14T09:14:06,596 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/2c9ea904770d418b858d7eb348a954b6 is 50, key is test_row_0/B:col10/1731575644128/Put/seqid=0 2024-11-14T09:14:06,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741936_1112 (size=13425) 2024-11-14T09:14:06,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741937_1113 (size=13425) 2024-11-14T09:14:06,616 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/b725a107ce9a4562b8637d60faf2628c as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/b725a107ce9a4562b8637d60faf2628c 2024-11-14T09:14:06,626 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a92123abacf02fce4f65c3ba94256e83/A of a92123abacf02fce4f65c3ba94256e83 into b725a107ce9a4562b8637d60faf2628c(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:14:06,626 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a92123abacf02fce4f65c3ba94256e83: 2024-11-14T09:14:06,627 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., storeName=a92123abacf02fce4f65c3ba94256e83/A, priority=13, startTime=1731575646582; duration=0sec 2024-11-14T09:14:06,627 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:14:06,627 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a92123abacf02fce4f65c3ba94256e83:A 2024-11-14T09:14:06,627 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:14:06,628 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:14:06,628 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): a92123abacf02fce4f65c3ba94256e83/C is initiating minor compaction (all files) 2024-11-14T09:14:06,628 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a92123abacf02fce4f65c3ba94256e83/C in TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:06,629 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/813a8032b4da43df9a70af6c6833cbbf, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/1fa947d9d8fe44baac4a79c508efd4f6, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/207cf17213694580878b44c1cbbd8edb] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp, totalSize=37.0 K 2024-11-14T09:14:06,629 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 813a8032b4da43df9a70af6c6833cbbf, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=856, earliestPutTs=1731575642793 2024-11-14T09:14:06,629 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1fa947d9d8fe44baac4a79c508efd4f6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=874, earliestPutTs=1731575642942 2024-11-14T09:14:06,630 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 207cf17213694580878b44c1cbbd8edb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=893, earliestPutTs=1731575644128 2024-11-14T09:14:06,638 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a92123abacf02fce4f65c3ba94256e83#C#compaction#101 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:14:06,639 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/268f696ece4b496f80187df6dea9a140 is 50, key is test_row_0/C:col10/1731575644128/Put/seqid=0 2024-11-14T09:14:06,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741938_1114 (size=13425) 2024-11-14T09:14:06,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-14T09:14:06,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:06,696 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing a92123abacf02fce4f65c3ba94256e83 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-14T09:14:06,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=A 2024-11-14T09:14:06,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:06,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=B 2024-11-14T09:14:06,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:06,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=C 2024-11-14T09:14:06,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:06,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/223c36fcad4c42cbb8a9d4f9d23c7b3e is 50, key is test_row_0/A:col10/1731575645329/Put/seqid=0 2024-11-14T09:14:06,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741939_1115 (size=9857) 2024-11-14T09:14:07,011 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/2c9ea904770d418b858d7eb348a954b6 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/2c9ea904770d418b858d7eb348a954b6 2024-11-14T09:14:07,027 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a92123abacf02fce4f65c3ba94256e83/B of a92123abacf02fce4f65c3ba94256e83 into 2c9ea904770d418b858d7eb348a954b6(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:14:07,027 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a92123abacf02fce4f65c3ba94256e83: 2024-11-14T09:14:07,027 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., storeName=a92123abacf02fce4f65c3ba94256e83/B, priority=13, startTime=1731575646583; duration=0sec 2024-11-14T09:14:07,027 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:07,027 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a92123abacf02fce4f65c3ba94256e83:B 2024-11-14T09:14:07,062 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/268f696ece4b496f80187df6dea9a140 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/268f696ece4b496f80187df6dea9a140 2024-11-14T09:14:07,072 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a92123abacf02fce4f65c3ba94256e83/C of a92123abacf02fce4f65c3ba94256e83 into 268f696ece4b496f80187df6dea9a140(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:14:07,073 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a92123abacf02fce4f65c3ba94256e83: 2024-11-14T09:14:07,073 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., storeName=a92123abacf02fce4f65c3ba94256e83/C, priority=13, startTime=1731575646583; duration=0sec 2024-11-14T09:14:07,073 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:07,073 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a92123abacf02fce4f65c3ba94256e83:C 2024-11-14T09:14:07,110 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=908 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/223c36fcad4c42cbb8a9d4f9d23c7b3e 2024-11-14T09:14:07,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/4427978f987a42ee880899c57ba70252 is 50, key is test_row_0/B:col10/1731575645329/Put/seqid=0 2024-11-14T09:14:07,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741940_1116 (size=9857) 2024-11-14T09:14:07,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-14T09:14:07,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on a92123abacf02fce4f65c3ba94256e83 2024-11-14T09:14:07,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. as already flushing 2024-11-14T09:14:07,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:07,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 308 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575707521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:07,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:07,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 322 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575707521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:07,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:07,529 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:07,529 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:07,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:07,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:07,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:07,540 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=908 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/4427978f987a42ee880899c57ba70252 2024-11-14T09:14:07,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/a2ce18fc32474a7588e45726c086483e is 50, key is test_row_0/C:col10/1731575645329/Put/seqid=0 2024-11-14T09:14:07,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741941_1117 (size=9857) 2024-11-14T09:14:07,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:07,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 309 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575707635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:07,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:07,638 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:07,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 323 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575707635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:07,638 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:07,638 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:07,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:07,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:07,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:07,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:07,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 310 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49938 deadline: 1731575707844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:07,846 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:07,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:07,846 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:07,846 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:07,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 324 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49950 deadline: 1731575707845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:07,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:07,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:07,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:07,915 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-14T09:14:07,915 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-14T09:14:07,915 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:14:07,915 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:14:07,916 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-14T09:14:07,916 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:14:07,917 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-14T09:14:07,918 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:14:07,920 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:14:07,920 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:14:07,920 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:14:07,920 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:14:07,921 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:14:07,921 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:14:07,921 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:14:07,921 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:14:07,922 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:14:07,922 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:14:07,922 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:14:07,922 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:14:07,960 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=908 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/a2ce18fc32474a7588e45726c086483e 2024-11-14T09:14:07,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/223c36fcad4c42cbb8a9d4f9d23c7b3e as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/223c36fcad4c42cbb8a9d4f9d23c7b3e 2024-11-14T09:14:07,970 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/223c36fcad4c42cbb8a9d4f9d23c7b3e, entries=100, sequenceid=908, filesize=9.6 K 2024-11-14T09:14:07,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/4427978f987a42ee880899c57ba70252 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/4427978f987a42ee880899c57ba70252 2024-11-14T09:14:07,975 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/4427978f987a42ee880899c57ba70252, entries=100, sequenceid=908, filesize=9.6 K 2024-11-14T09:14:07,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/a2ce18fc32474a7588e45726c086483e as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/a2ce18fc32474a7588e45726c086483e 2024-11-14T09:14:07,980 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/a2ce18fc32474a7588e45726c086483e, entries=100, sequenceid=908, filesize=9.6 K 2024-11-14T09:14:07,981 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for a92123abacf02fce4f65c3ba94256e83 in 1285ms, sequenceid=908, compaction requested=false 2024-11-14T09:14:07,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for a92123abacf02fce4f65c3ba94256e83: 2024-11-14T09:14:07,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:07,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-14T09:14:07,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-14T09:14:07,984 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-14T09:14:07,984 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8230 sec 2024-11-14T09:14:07,986 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 2.8280 sec 2024-11-14T09:14:08,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on a92123abacf02fce4f65c3ba94256e83 2024-11-14T09:14:08,175 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a92123abacf02fce4f65c3ba94256e83 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-14T09:14:08,175 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-14T09:14:08,175 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-14T09:14:08,175 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:14:08,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=A 2024-11-14T09:14:08,175 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:14:08,175 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:14:08,175 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:14:08,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:14:08,176 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:14:08,176 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:14:08,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=B 2024-11-14T09:14:08,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:14:08,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=C 2024-11-14T09:14:08,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:14:08,176 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:14:08,177 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:14:08,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/9b2a0f4a26ba4154a8878ff0d79dac91 is 50, key is test_row_0/A:col10/1731575648155/Put/seqid=0 2024-11-14T09:14:08,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741942_1118 (size=12301) 2024-11-14T09:14:08,587 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=931 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/9b2a0f4a26ba4154a8878ff0d79dac91 2024-11-14T09:14:08,598 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/9f4688cd035f46ccafa65755753b27ea is 50, key is test_row_0/B:col10/1731575648155/Put/seqid=0 2024-11-14T09:14:08,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741943_1119 (size=12301) 2024-11-14T09:14:08,607 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=931 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/9f4688cd035f46ccafa65755753b27ea 2024-11-14T09:14:08,616 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/c3e905a149ea40a3bba7429af68308e8 is 50, key is test_row_0/C:col10/1731575648155/Put/seqid=0 2024-11-14T09:14:08,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741944_1120 (size=12301) 2024-11-14T09:14:09,025 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=931 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/c3e905a149ea40a3bba7429af68308e8 2024-11-14T09:14:09,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/9b2a0f4a26ba4154a8878ff0d79dac91 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/9b2a0f4a26ba4154a8878ff0d79dac91 2024-11-14T09:14:09,038 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/9b2a0f4a26ba4154a8878ff0d79dac91, entries=150, sequenceid=931, filesize=12.0 K 2024-11-14T09:14:09,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/9f4688cd035f46ccafa65755753b27ea as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/9f4688cd035f46ccafa65755753b27ea 2024-11-14T09:14:09,045 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/9f4688cd035f46ccafa65755753b27ea, entries=150, sequenceid=931, filesize=12.0 K 2024-11-14T09:14:09,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/c3e905a149ea40a3bba7429af68308e8 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/c3e905a149ea40a3bba7429af68308e8 2024-11-14T09:14:09,053 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/c3e905a149ea40a3bba7429af68308e8, entries=150, sequenceid=931, filesize=12.0 K 2024-11-14T09:14:09,054 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=0 B/0 for a92123abacf02fce4f65c3ba94256e83 in 879ms, sequenceid=931, compaction requested=true 2024-11-14T09:14:09,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a92123abacf02fce4f65c3ba94256e83: 2024-11-14T09:14:09,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a92123abacf02fce4f65c3ba94256e83:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:14:09,054 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:14:09,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:09,054 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:14:09,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a92123abacf02fce4f65c3ba94256e83:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:14:09,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:09,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a92123abacf02fce4f65c3ba94256e83:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:14:09,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:14:09,056 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35583 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:14:09,056 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): a92123abacf02fce4f65c3ba94256e83/A is initiating minor compaction (all files) 2024-11-14T09:14:09,056 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a92123abacf02fce4f65c3ba94256e83/A in TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:09,056 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/b725a107ce9a4562b8637d60faf2628c, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/223c36fcad4c42cbb8a9d4f9d23c7b3e, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/9b2a0f4a26ba4154a8878ff0d79dac91] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp, totalSize=34.7 K 2024-11-14T09:14:09,056 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35583 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:14:09,057 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): a92123abacf02fce4f65c3ba94256e83/B is initiating minor compaction (all files) 2024-11-14T09:14:09,057 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a92123abacf02fce4f65c3ba94256e83/B in TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:09,057 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/2c9ea904770d418b858d7eb348a954b6, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/4427978f987a42ee880899c57ba70252, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/9f4688cd035f46ccafa65755753b27ea] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp, totalSize=34.7 K 2024-11-14T09:14:09,057 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting b725a107ce9a4562b8637d60faf2628c, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=893, earliestPutTs=1731575644128 2024-11-14T09:14:09,057 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c9ea904770d418b858d7eb348a954b6, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=893, earliestPutTs=1731575644128 2024-11-14T09:14:09,058 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 223c36fcad4c42cbb8a9d4f9d23c7b3e, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=908, earliestPutTs=1731575645329 2024-11-14T09:14:09,058 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 4427978f987a42ee880899c57ba70252, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=908, earliestPutTs=1731575645329 2024-11-14T09:14:09,058 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b2a0f4a26ba4154a8878ff0d79dac91, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=931, earliestPutTs=1731575647515 2024-11-14T09:14:09,059 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f4688cd035f46ccafa65755753b27ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=931, earliestPutTs=1731575647515 2024-11-14T09:14:09,074 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a92123abacf02fce4f65c3ba94256e83#B#compaction#108 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:14:09,075 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/baffaefdb3f244b2bd770626ce331c26 is 50, key is test_row_0/B:col10/1731575648155/Put/seqid=0 2024-11-14T09:14:09,083 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a92123abacf02fce4f65c3ba94256e83#A#compaction#109 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:14:09,084 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/525a1f337e2546eaa493880a8319619b is 50, key is test_row_0/A:col10/1731575648155/Put/seqid=0 2024-11-14T09:14:09,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741945_1121 (size=13527) 2024-11-14T09:14:09,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741946_1122 (size=13527) 2024-11-14T09:14:09,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-14T09:14:09,335 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:14:09,506 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/baffaefdb3f244b2bd770626ce331c26 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/baffaefdb3f244b2bd770626ce331c26 2024-11-14T09:14:09,511 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/525a1f337e2546eaa493880a8319619b as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/525a1f337e2546eaa493880a8319619b 2024-11-14T09:14:09,515 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a92123abacf02fce4f65c3ba94256e83/B of a92123abacf02fce4f65c3ba94256e83 into baffaefdb3f244b2bd770626ce331c26(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:14:09,515 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a92123abacf02fce4f65c3ba94256e83: 2024-11-14T09:14:09,515 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., storeName=a92123abacf02fce4f65c3ba94256e83/B, priority=13, startTime=1731575649054; duration=0sec 2024-11-14T09:14:09,515 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:14:09,516 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a92123abacf02fce4f65c3ba94256e83:B 2024-11-14T09:14:09,516 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:14:09,517 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35583 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:14:09,517 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): a92123abacf02fce4f65c3ba94256e83/C is initiating minor compaction (all files) 2024-11-14T09:14:09,517 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a92123abacf02fce4f65c3ba94256e83/C in TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:09,517 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/268f696ece4b496f80187df6dea9a140, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/a2ce18fc32474a7588e45726c086483e, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/c3e905a149ea40a3bba7429af68308e8] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp, totalSize=34.7 K 2024-11-14T09:14:09,518 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 268f696ece4b496f80187df6dea9a140, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=893, earliestPutTs=1731575644128 2024-11-14T09:14:09,518 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting a2ce18fc32474a7588e45726c086483e, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=908, earliestPutTs=1731575645329 2024-11-14T09:14:09,519 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting c3e905a149ea40a3bba7429af68308e8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=931, earliestPutTs=1731575647515 2024-11-14T09:14:09,520 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a92123abacf02fce4f65c3ba94256e83/A of a92123abacf02fce4f65c3ba94256e83 into 525a1f337e2546eaa493880a8319619b(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:14:09,520 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a92123abacf02fce4f65c3ba94256e83: 2024-11-14T09:14:09,520 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., storeName=a92123abacf02fce4f65c3ba94256e83/A, priority=13, startTime=1731575649054; duration=0sec 2024-11-14T09:14:09,520 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:09,520 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a92123abacf02fce4f65c3ba94256e83:A 2024-11-14T09:14:09,530 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a92123abacf02fce4f65c3ba94256e83#C#compaction#110 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:14:09,531 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/b08b0e2604fb42609a71b247977501c7 is 50, key is test_row_0/C:col10/1731575648155/Put/seqid=0 2024-11-14T09:14:09,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741947_1123 (size=13527) 2024-11-14T09:14:09,545 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/b08b0e2604fb42609a71b247977501c7 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/b08b0e2604fb42609a71b247977501c7 2024-11-14T09:14:09,552 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a92123abacf02fce4f65c3ba94256e83/C of a92123abacf02fce4f65c3ba94256e83 into b08b0e2604fb42609a71b247977501c7(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:14:09,553 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a92123abacf02fce4f65c3ba94256e83: 2024-11-14T09:14:09,553 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., storeName=a92123abacf02fce4f65c3ba94256e83/C, priority=13, startTime=1731575649055; duration=0sec 2024-11-14T09:14:09,553 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:09,553 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a92123abacf02fce4f65c3ba94256e83:C 2024-11-14T09:14:10,753 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T09:14:11,887 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-14T09:14:11,887 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-14T09:14:11,887 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:14:11,887 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:14:11,887 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:14:11,887 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:14:11,887 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:14:11,887 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:14:11,887 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:14:11,887 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:14:11,896 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-14T09:14:11,897 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:14:11,897 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:14:11,897 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:14:11,897 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:14:11,897 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T09:14:11,897 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 103 2024-11-14T09:14:11,897 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 248 2024-11-14T09:14:11,897 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 262 2024-11-14T09:14:11,897 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 94 2024-11-14T09:14:11,897 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 118 2024-11-14T09:14:11,897 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T09:14:11,897 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2617 2024-11-14T09:14:11,897 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2864 2024-11-14T09:14:11,897 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T09:14:11,897 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1203 2024-11-14T09:14:11,897 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3605 rows 2024-11-14T09:14:11,897 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1196 2024-11-14T09:14:11,897 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3584 rows 2024-11-14T09:14:11,897 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T09:14:11,897 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T09:14:11,898 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:14:11,898 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:14:11,898 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:14:11,902 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-14T09:14:11,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T09:14:11,911 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731575651911"}]},"ts":"1731575651911"} 2024-11-14T09:14:11,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-14T09:14:11,913 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T09:14:11,913 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T09:14:11,914 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T09:14:11,918 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a92123abacf02fce4f65c3ba94256e83, UNASSIGN}] 2024-11-14T09:14:11,919 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a92123abacf02fce4f65c3ba94256e83, UNASSIGN 2024-11-14T09:14:11,920 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=a92123abacf02fce4f65c3ba94256e83, regionState=CLOSING, regionLocation=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:11,922 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a92123abacf02fce4f65c3ba94256e83, UNASSIGN because future has completed 2024-11-14T09:14:11,923 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T09:14:11,923 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393}] 2024-11-14T09:14:12,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-14T09:14:12,079 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(122): Close a92123abacf02fce4f65c3ba94256e83 2024-11-14T09:14:12,079 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T09:14:12,080 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1722): Closing a92123abacf02fce4f65c3ba94256e83, disabling compactions & flushes 2024-11-14T09:14:12,080 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:12,080 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:12,080 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. after waiting 0 ms 2024-11-14T09:14:12,080 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:12,080 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(2902): Flushing a92123abacf02fce4f65c3ba94256e83 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-14T09:14:12,080 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=A 2024-11-14T09:14:12,081 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:12,081 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=B 2024-11-14T09:14:12,081 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:12,081 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a92123abacf02fce4f65c3ba94256e83, store=C 2024-11-14T09:14:12,081 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:12,085 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/cd203ee7356a4456b65eb96359750cb1 is 50, key is test_row_0/A:col10/1731575651885/Put/seqid=0 2024-11-14T09:14:12,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741948_1124 (size=9857) 2024-11-14T09:14:12,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-14T09:14:12,490 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=940 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/cd203ee7356a4456b65eb96359750cb1 2024-11-14T09:14:12,498 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/2f9648d32d224845a175b0244d0cb058 is 50, key is test_row_0/B:col10/1731575651885/Put/seqid=0 2024-11-14T09:14:12,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741949_1125 (size=9857) 2024-11-14T09:14:12,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-14T09:14:12,902 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=940 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/2f9648d32d224845a175b0244d0cb058 2024-11-14T09:14:12,910 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/c01befaff87e4fb58f9c8c98bce34893 is 50, key is test_row_0/C:col10/1731575651885/Put/seqid=0 2024-11-14T09:14:12,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741950_1126 (size=9857) 2024-11-14T09:14:13,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-14T09:14:13,314 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=940 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/c01befaff87e4fb58f9c8c98bce34893 2024-11-14T09:14:13,319 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/A/cd203ee7356a4456b65eb96359750cb1 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/cd203ee7356a4456b65eb96359750cb1 2024-11-14T09:14:13,324 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/cd203ee7356a4456b65eb96359750cb1, entries=100, sequenceid=940, filesize=9.6 K 2024-11-14T09:14:13,325 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/B/2f9648d32d224845a175b0244d0cb058 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/2f9648d32d224845a175b0244d0cb058 2024-11-14T09:14:13,330 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/2f9648d32d224845a175b0244d0cb058, entries=100, sequenceid=940, filesize=9.6 K 2024-11-14T09:14:13,331 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/.tmp/C/c01befaff87e4fb58f9c8c98bce34893 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/c01befaff87e4fb58f9c8c98bce34893 2024-11-14T09:14:13,335 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/c01befaff87e4fb58f9c8c98bce34893, entries=100, sequenceid=940, filesize=9.6 K 2024-11-14T09:14:13,337 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for a92123abacf02fce4f65c3ba94256e83 in 1256ms, sequenceid=940, compaction requested=false 2024-11-14T09:14:13,337 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/48e3bdfc754c47d9a827e09d90ec10e5, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/c309c9d7673144ec8c135c77844a4b17, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/9ab6d0155d964562a5cd5b4716cc8bff, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/4f384c57c2474db4a6386afdf5ad5ef9, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/690bcafabc6548f99789cd73462022ef, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/257325ccf4b24fe7b5cc16126b3f2798, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/0da76ca844b54096b8bd1786dd0a979e, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/02500cc37f6041a8ab34a44f4f8b7081, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/4ca60702129b4adaa7de671c1e7f13e8, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/98c612af874e4e73b80d84397de05bb2, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/40af6745cad845d2a168d961a77100f7, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/1ef08393e3284940ab6e0c74c5403042, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/41f9beb335a24305902272a41cb276d1, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/93d10d3b98154fdfa4e8e43a0d8fb0d5, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/5345edf9034d4c1495348e1c41d7a839, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/d679062c9b5847b88ec3026d0c854385, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/e528b16185a44ed8b4eeeaa5fbe398e8, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/14fdd7793cda4a36a5d18f42447d52c2, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/b18068da669940cd99ba9435673f3b02, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/f22c370481564b6a88e5bb3387b9ae4f, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/8afdaad81e414b98b9f75c4a44cb7b6a, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/96ea282569d94124b9bf0cfa01b17e5b, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/d1ed19fcc97b45829d8cd7bfacee640f, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/12e846cdd6574f86a2eca6a8aaca5ef2, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/0fa5eb9b15c94bb789b09a316273579d, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/0ada844e0ac24df2a410bcaaee5e44d0, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/a9d1625c16164fd69c396896465cfd64, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/10247dd911384f3c815184f3272ed175, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/8d353be469204ab090c0315eb9c94dd6, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/0a6cbc5477cb44ecb7c7b912a6e03f4f, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/99819218d62a44e5855eae38eb83d3aa, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/141a797724fb4576b5a126957eec4a1b, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/b725a107ce9a4562b8637d60faf2628c, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/459fbb7015be49a49f2ebc9a1303c34c, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/223c36fcad4c42cbb8a9d4f9d23c7b3e, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/9b2a0f4a26ba4154a8878ff0d79dac91] to archive 2024-11-14T09:14:13,340 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T09:14:13,345 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/48e3bdfc754c47d9a827e09d90ec10e5 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/48e3bdfc754c47d9a827e09d90ec10e5 2024-11-14T09:14:13,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/c309c9d7673144ec8c135c77844a4b17 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/c309c9d7673144ec8c135c77844a4b17 2024-11-14T09:14:13,348 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/9ab6d0155d964562a5cd5b4716cc8bff to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/9ab6d0155d964562a5cd5b4716cc8bff 2024-11-14T09:14:13,349 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/4f384c57c2474db4a6386afdf5ad5ef9 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/4f384c57c2474db4a6386afdf5ad5ef9 2024-11-14T09:14:13,351 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/690bcafabc6548f99789cd73462022ef to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/690bcafabc6548f99789cd73462022ef 2024-11-14T09:14:13,352 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/257325ccf4b24fe7b5cc16126b3f2798 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/257325ccf4b24fe7b5cc16126b3f2798 2024-11-14T09:14:13,353 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/0da76ca844b54096b8bd1786dd0a979e to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/0da76ca844b54096b8bd1786dd0a979e 2024-11-14T09:14:13,354 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/02500cc37f6041a8ab34a44f4f8b7081 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/02500cc37f6041a8ab34a44f4f8b7081 2024-11-14T09:14:13,355 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/4ca60702129b4adaa7de671c1e7f13e8 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/4ca60702129b4adaa7de671c1e7f13e8 2024-11-14T09:14:13,356 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/98c612af874e4e73b80d84397de05bb2 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/98c612af874e4e73b80d84397de05bb2 2024-11-14T09:14:13,358 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/40af6745cad845d2a168d961a77100f7 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/40af6745cad845d2a168d961a77100f7 2024-11-14T09:14:13,359 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/1ef08393e3284940ab6e0c74c5403042 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/1ef08393e3284940ab6e0c74c5403042 2024-11-14T09:14:13,360 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/41f9beb335a24305902272a41cb276d1 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/41f9beb335a24305902272a41cb276d1 2024-11-14T09:14:13,361 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/93d10d3b98154fdfa4e8e43a0d8fb0d5 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/93d10d3b98154fdfa4e8e43a0d8fb0d5 2024-11-14T09:14:13,362 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/5345edf9034d4c1495348e1c41d7a839 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/5345edf9034d4c1495348e1c41d7a839 2024-11-14T09:14:13,364 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/d679062c9b5847b88ec3026d0c854385 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/d679062c9b5847b88ec3026d0c854385 2024-11-14T09:14:13,365 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/e528b16185a44ed8b4eeeaa5fbe398e8 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/e528b16185a44ed8b4eeeaa5fbe398e8 2024-11-14T09:14:13,366 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/14fdd7793cda4a36a5d18f42447d52c2 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/14fdd7793cda4a36a5d18f42447d52c2 2024-11-14T09:14:13,367 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/b18068da669940cd99ba9435673f3b02 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/b18068da669940cd99ba9435673f3b02 2024-11-14T09:14:13,368 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/f22c370481564b6a88e5bb3387b9ae4f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/f22c370481564b6a88e5bb3387b9ae4f 2024-11-14T09:14:13,370 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/8afdaad81e414b98b9f75c4a44cb7b6a to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/8afdaad81e414b98b9f75c4a44cb7b6a 2024-11-14T09:14:13,371 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/96ea282569d94124b9bf0cfa01b17e5b to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/96ea282569d94124b9bf0cfa01b17e5b 2024-11-14T09:14:13,372 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/d1ed19fcc97b45829d8cd7bfacee640f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/d1ed19fcc97b45829d8cd7bfacee640f 2024-11-14T09:14:13,373 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/12e846cdd6574f86a2eca6a8aaca5ef2 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/12e846cdd6574f86a2eca6a8aaca5ef2 2024-11-14T09:14:13,374 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/0fa5eb9b15c94bb789b09a316273579d to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/0fa5eb9b15c94bb789b09a316273579d 2024-11-14T09:14:13,375 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/0ada844e0ac24df2a410bcaaee5e44d0 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/0ada844e0ac24df2a410bcaaee5e44d0 2024-11-14T09:14:13,376 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/a9d1625c16164fd69c396896465cfd64 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/a9d1625c16164fd69c396896465cfd64 2024-11-14T09:14:13,377 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/10247dd911384f3c815184f3272ed175 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/10247dd911384f3c815184f3272ed175 2024-11-14T09:14:13,379 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/8d353be469204ab090c0315eb9c94dd6 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/8d353be469204ab090c0315eb9c94dd6 2024-11-14T09:14:13,380 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/0a6cbc5477cb44ecb7c7b912a6e03f4f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/0a6cbc5477cb44ecb7c7b912a6e03f4f 2024-11-14T09:14:13,381 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/99819218d62a44e5855eae38eb83d3aa to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/99819218d62a44e5855eae38eb83d3aa 2024-11-14T09:14:13,382 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/141a797724fb4576b5a126957eec4a1b to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/141a797724fb4576b5a126957eec4a1b 2024-11-14T09:14:13,383 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/b725a107ce9a4562b8637d60faf2628c to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/b725a107ce9a4562b8637d60faf2628c 2024-11-14T09:14:13,384 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/459fbb7015be49a49f2ebc9a1303c34c to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/459fbb7015be49a49f2ebc9a1303c34c 2024-11-14T09:14:13,385 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/223c36fcad4c42cbb8a9d4f9d23c7b3e to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/223c36fcad4c42cbb8a9d4f9d23c7b3e 2024-11-14T09:14:13,386 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/9b2a0f4a26ba4154a8878ff0d79dac91 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/9b2a0f4a26ba4154a8878ff0d79dac91 2024-11-14T09:14:13,399 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/9a5626a483f944d494ac17692769a729, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/e7cacf86dde84ea0ae65d6fb0d70417b, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/3e17093d314248e49c7cc0f738ad3e5f, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/cee7ef0618e4446ca07773cc4f4bb2f6, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/069b66ade5cf4261ac1a23824628912d, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/c6a295c0796145909cbb0ed78cfcb54b, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/7c5e02155102424e9cc8eb1e4ab6e209, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/f72ffa4260b643c292836c07077703ec, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/f0499f50d9494bf199c90136c45743d9, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/376f2e79eb3c4908b44d2a11fe964418, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/a1ca9c7f90b6432f836343497117e2ca, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/0c8bbd0bde3d45b398f632cff954645d, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/6770f19af96c42de8198cef9aaa3af4c, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/8e94fe9abb0d4cd98fd121a4c1b5f2c7, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/776de273b5d441c89ee4d88c9ed9c667, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/db6c2237c6064af6872c8ebd158193c7, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/6f26b8f249f24341bea45680bb9aeca0, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/65bc053b0c49432484d497989ec5e158, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/e3c814027e47479488c625531b0be89a, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/b9eb900add4142189ff8f6be0fc0f53e, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/2a83db7c6d144326a946f31abaedd4fb, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/bde88938d36a44e7b75761ceb01de4e3, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/38285b08250d4fe8a5aa7c6ce77227e9, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/e10190d283024a0195e408f393d5937e, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/f613b1d955e24f5fb154ba100a9766d4, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/d18da34ee95d44758e3fca3dafd2a9b7, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/2debd4527a894a8e98ff11108a7f8066, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/f26daec2a3ba4645a31e89ba7df5a719, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/aeb2ce283ea34d63a8c1f85238832986, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/4adfc20ad13443079b8e9ad2f5ddcc09, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/d8623bdb4fd141dc98d37c0930f077dd, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/16d44bf4b50d4f4b84e5f5e31a341c6b, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/2c9ea904770d418b858d7eb348a954b6, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/584e999ed58c4fb7b78fa02548daa987, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/4427978f987a42ee880899c57ba70252, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/9f4688cd035f46ccafa65755753b27ea] to archive 2024-11-14T09:14:13,400 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T09:14:13,402 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/9a5626a483f944d494ac17692769a729 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/9a5626a483f944d494ac17692769a729 2024-11-14T09:14:13,403 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/e7cacf86dde84ea0ae65d6fb0d70417b to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/e7cacf86dde84ea0ae65d6fb0d70417b 2024-11-14T09:14:13,404 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/3e17093d314248e49c7cc0f738ad3e5f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/3e17093d314248e49c7cc0f738ad3e5f 2024-11-14T09:14:13,405 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/cee7ef0618e4446ca07773cc4f4bb2f6 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/cee7ef0618e4446ca07773cc4f4bb2f6 2024-11-14T09:14:13,406 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/069b66ade5cf4261ac1a23824628912d to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/069b66ade5cf4261ac1a23824628912d 2024-11-14T09:14:13,407 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/c6a295c0796145909cbb0ed78cfcb54b to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/c6a295c0796145909cbb0ed78cfcb54b 2024-11-14T09:14:13,408 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/7c5e02155102424e9cc8eb1e4ab6e209 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/7c5e02155102424e9cc8eb1e4ab6e209 2024-11-14T09:14:13,409 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/f72ffa4260b643c292836c07077703ec to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/f72ffa4260b643c292836c07077703ec 2024-11-14T09:14:13,410 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/f0499f50d9494bf199c90136c45743d9 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/f0499f50d9494bf199c90136c45743d9 2024-11-14T09:14:13,411 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/376f2e79eb3c4908b44d2a11fe964418 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/376f2e79eb3c4908b44d2a11fe964418 2024-11-14T09:14:13,412 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/a1ca9c7f90b6432f836343497117e2ca to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/a1ca9c7f90b6432f836343497117e2ca 2024-11-14T09:14:13,414 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/0c8bbd0bde3d45b398f632cff954645d to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/0c8bbd0bde3d45b398f632cff954645d 2024-11-14T09:14:13,415 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/6770f19af96c42de8198cef9aaa3af4c to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/6770f19af96c42de8198cef9aaa3af4c 2024-11-14T09:14:13,416 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/8e94fe9abb0d4cd98fd121a4c1b5f2c7 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/8e94fe9abb0d4cd98fd121a4c1b5f2c7 2024-11-14T09:14:13,417 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/776de273b5d441c89ee4d88c9ed9c667 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/776de273b5d441c89ee4d88c9ed9c667 2024-11-14T09:14:13,418 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/db6c2237c6064af6872c8ebd158193c7 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/db6c2237c6064af6872c8ebd158193c7 2024-11-14T09:14:13,419 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/6f26b8f249f24341bea45680bb9aeca0 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/6f26b8f249f24341bea45680bb9aeca0 2024-11-14T09:14:13,421 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/65bc053b0c49432484d497989ec5e158 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/65bc053b0c49432484d497989ec5e158 2024-11-14T09:14:13,422 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/e3c814027e47479488c625531b0be89a to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/e3c814027e47479488c625531b0be89a 2024-11-14T09:14:13,423 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/b9eb900add4142189ff8f6be0fc0f53e to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/b9eb900add4142189ff8f6be0fc0f53e 2024-11-14T09:14:13,424 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/2a83db7c6d144326a946f31abaedd4fb to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/2a83db7c6d144326a946f31abaedd4fb 2024-11-14T09:14:13,426 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/bde88938d36a44e7b75761ceb01de4e3 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/bde88938d36a44e7b75761ceb01de4e3 2024-11-14T09:14:13,427 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/38285b08250d4fe8a5aa7c6ce77227e9 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/38285b08250d4fe8a5aa7c6ce77227e9 2024-11-14T09:14:13,428 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/e10190d283024a0195e408f393d5937e to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/e10190d283024a0195e408f393d5937e 2024-11-14T09:14:13,429 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/f613b1d955e24f5fb154ba100a9766d4 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/f613b1d955e24f5fb154ba100a9766d4 2024-11-14T09:14:13,430 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/d18da34ee95d44758e3fca3dafd2a9b7 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/d18da34ee95d44758e3fca3dafd2a9b7 2024-11-14T09:14:13,431 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/2debd4527a894a8e98ff11108a7f8066 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/2debd4527a894a8e98ff11108a7f8066 2024-11-14T09:14:13,432 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/f26daec2a3ba4645a31e89ba7df5a719 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/f26daec2a3ba4645a31e89ba7df5a719 2024-11-14T09:14:13,433 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/aeb2ce283ea34d63a8c1f85238832986 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/aeb2ce283ea34d63a8c1f85238832986 2024-11-14T09:14:13,435 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/4adfc20ad13443079b8e9ad2f5ddcc09 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/4adfc20ad13443079b8e9ad2f5ddcc09 2024-11-14T09:14:13,436 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/d8623bdb4fd141dc98d37c0930f077dd to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/d8623bdb4fd141dc98d37c0930f077dd 2024-11-14T09:14:13,437 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/16d44bf4b50d4f4b84e5f5e31a341c6b to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/16d44bf4b50d4f4b84e5f5e31a341c6b 2024-11-14T09:14:13,438 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/2c9ea904770d418b858d7eb348a954b6 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/2c9ea904770d418b858d7eb348a954b6 2024-11-14T09:14:13,439 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/584e999ed58c4fb7b78fa02548daa987 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/584e999ed58c4fb7b78fa02548daa987 2024-11-14T09:14:13,440 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/4427978f987a42ee880899c57ba70252 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/4427978f987a42ee880899c57ba70252 2024-11-14T09:14:13,442 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/9f4688cd035f46ccafa65755753b27ea to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/9f4688cd035f46ccafa65755753b27ea 2024-11-14T09:14:13,443 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/ac7dc2dcde354dedb5bf0b92634c7b9a, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/9131cf1925354ac0b1a2c87dfa49ef14, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/fe0d594cf40d467d9b9b939c49abfcf4, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/9c83ad8d4dad4ae0bb34183f73968c7d, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/5e96899ed6334936ad9c2327c711e028, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/e21e157f4b5040d9bfc7d8e68adbffa7, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/b5b75880497b4eaaaf77b253d910ef77, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/515ce2d699ac46fdbe1e9dcc6d4c1918, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/60b218fdadf9459b81501264fc7f517d, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/2a0afeec1f044ef3aa64c408e0b8a1a5, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/19f0b5abff5344239b2d88b38d790861, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/d91dd276de3a498caf953e63ece2dc5d, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/5fcc5458fae24cb3884544764d1e8421, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/0223fb546a4f495fbcf4c159d5ad89c2, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/f5089ca6fd9e424199d97f3e8d54bea3, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/fe1c391d3288430d8e1c3f2a31d451e6, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/09d3ef601e3a448d8dfd5d19ff515880, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/4353b58396794073802ea808c8421935, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/4988ffdf37af48f3b23acd289a522449, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/644a522b98d24fe7a1367d430c9dc738, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/9798e58624a94992808a8020bd535514, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/4c53cf2a372547cf9720c5335ae34ac4, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/9a8d65c63adf45b5bdfac78669c2378a, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/6145abc98fa94e4a82bf9728ddb68e15, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/fb4bb9e885bb433c8ef2d1293812f174, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/7057ca4cc01d4276b9f7ab1aedfdcb07, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/00fc75d98be34de59fe98a9173cd9780, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/69edb8844b8044079dca1ff888d4aba6, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/ce3cf758e9f840899c0394c939f03495, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/813a8032b4da43df9a70af6c6833cbbf, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/07d3f5dc69cc4c61b17ccd3f22c85625, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/1fa947d9d8fe44baac4a79c508efd4f6, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/268f696ece4b496f80187df6dea9a140, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/207cf17213694580878b44c1cbbd8edb, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/a2ce18fc32474a7588e45726c086483e, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/c3e905a149ea40a3bba7429af68308e8] to archive 2024-11-14T09:14:13,444 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T09:14:13,446 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/ac7dc2dcde354dedb5bf0b92634c7b9a to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/ac7dc2dcde354dedb5bf0b92634c7b9a 2024-11-14T09:14:13,447 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/9131cf1925354ac0b1a2c87dfa49ef14 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/9131cf1925354ac0b1a2c87dfa49ef14 2024-11-14T09:14:13,448 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/fe0d594cf40d467d9b9b939c49abfcf4 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/fe0d594cf40d467d9b9b939c49abfcf4 2024-11-14T09:14:13,449 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/9c83ad8d4dad4ae0bb34183f73968c7d to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/9c83ad8d4dad4ae0bb34183f73968c7d 2024-11-14T09:14:13,451 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/5e96899ed6334936ad9c2327c711e028 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/5e96899ed6334936ad9c2327c711e028 2024-11-14T09:14:13,452 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/e21e157f4b5040d9bfc7d8e68adbffa7 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/e21e157f4b5040d9bfc7d8e68adbffa7 2024-11-14T09:14:13,453 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/b5b75880497b4eaaaf77b253d910ef77 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/b5b75880497b4eaaaf77b253d910ef77 2024-11-14T09:14:13,454 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/515ce2d699ac46fdbe1e9dcc6d4c1918 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/515ce2d699ac46fdbe1e9dcc6d4c1918 2024-11-14T09:14:13,455 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/60b218fdadf9459b81501264fc7f517d to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/60b218fdadf9459b81501264fc7f517d 2024-11-14T09:14:13,456 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/2a0afeec1f044ef3aa64c408e0b8a1a5 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/2a0afeec1f044ef3aa64c408e0b8a1a5 2024-11-14T09:14:13,457 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/19f0b5abff5344239b2d88b38d790861 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/19f0b5abff5344239b2d88b38d790861 2024-11-14T09:14:13,458 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/d91dd276de3a498caf953e63ece2dc5d to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/d91dd276de3a498caf953e63ece2dc5d 2024-11-14T09:14:13,459 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/5fcc5458fae24cb3884544764d1e8421 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/5fcc5458fae24cb3884544764d1e8421 2024-11-14T09:14:13,460 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/0223fb546a4f495fbcf4c159d5ad89c2 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/0223fb546a4f495fbcf4c159d5ad89c2 2024-11-14T09:14:13,462 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/f5089ca6fd9e424199d97f3e8d54bea3 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/f5089ca6fd9e424199d97f3e8d54bea3 2024-11-14T09:14:13,463 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/fe1c391d3288430d8e1c3f2a31d451e6 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/fe1c391d3288430d8e1c3f2a31d451e6 2024-11-14T09:14:13,464 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/09d3ef601e3a448d8dfd5d19ff515880 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/09d3ef601e3a448d8dfd5d19ff515880 2024-11-14T09:14:13,465 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/4353b58396794073802ea808c8421935 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/4353b58396794073802ea808c8421935 2024-11-14T09:14:13,466 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/4988ffdf37af48f3b23acd289a522449 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/4988ffdf37af48f3b23acd289a522449 2024-11-14T09:14:13,467 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/644a522b98d24fe7a1367d430c9dc738 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/644a522b98d24fe7a1367d430c9dc738 2024-11-14T09:14:13,468 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/9798e58624a94992808a8020bd535514 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/9798e58624a94992808a8020bd535514 2024-11-14T09:14:13,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/4c53cf2a372547cf9720c5335ae34ac4 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/4c53cf2a372547cf9720c5335ae34ac4 2024-11-14T09:14:13,470 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/9a8d65c63adf45b5bdfac78669c2378a to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/9a8d65c63adf45b5bdfac78669c2378a 2024-11-14T09:14:13,471 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/6145abc98fa94e4a82bf9728ddb68e15 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/6145abc98fa94e4a82bf9728ddb68e15 2024-11-14T09:14:13,472 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/fb4bb9e885bb433c8ef2d1293812f174 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/fb4bb9e885bb433c8ef2d1293812f174 2024-11-14T09:14:13,473 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/7057ca4cc01d4276b9f7ab1aedfdcb07 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/7057ca4cc01d4276b9f7ab1aedfdcb07 2024-11-14T09:14:13,474 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/00fc75d98be34de59fe98a9173cd9780 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/00fc75d98be34de59fe98a9173cd9780 2024-11-14T09:14:13,475 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/69edb8844b8044079dca1ff888d4aba6 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/69edb8844b8044079dca1ff888d4aba6 2024-11-14T09:14:13,476 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/ce3cf758e9f840899c0394c939f03495 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/ce3cf758e9f840899c0394c939f03495 2024-11-14T09:14:13,477 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/813a8032b4da43df9a70af6c6833cbbf to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/813a8032b4da43df9a70af6c6833cbbf 2024-11-14T09:14:13,478 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/07d3f5dc69cc4c61b17ccd3f22c85625 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/07d3f5dc69cc4c61b17ccd3f22c85625 2024-11-14T09:14:13,479 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/1fa947d9d8fe44baac4a79c508efd4f6 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/1fa947d9d8fe44baac4a79c508efd4f6 2024-11-14T09:14:13,480 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/268f696ece4b496f80187df6dea9a140 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/268f696ece4b496f80187df6dea9a140 2024-11-14T09:14:13,481 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/207cf17213694580878b44c1cbbd8edb to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/207cf17213694580878b44c1cbbd8edb 2024-11-14T09:14:13,482 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/a2ce18fc32474a7588e45726c086483e to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/a2ce18fc32474a7588e45726c086483e 2024-11-14T09:14:13,484 DEBUG [StoreCloser-TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/c3e905a149ea40a3bba7429af68308e8 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/c3e905a149ea40a3bba7429af68308e8 2024-11-14T09:14:13,489 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/recovered.edits/943.seqid, newMaxSeqId=943, maxSeqId=1 2024-11-14T09:14:13,489 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. 2024-11-14T09:14:13,489 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1676): Region close journal for a92123abacf02fce4f65c3ba94256e83: Waiting for close lock at 1731575652080Running coprocessor pre-close hooks at 1731575652080Disabling compacts and flushes for region at 1731575652080Disabling writes for close at 1731575652080Obtaining lock to block concurrent updates at 1731575652080Preparing flush snapshotting stores in a92123abacf02fce4f65c3ba94256e83 at 1731575652080Finished memstore snapshotting TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1731575652081 (+1 ms)Flushing stores of TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83. at 1731575652081Flushing a92123abacf02fce4f65c3ba94256e83/A: creating writer at 1731575652081Flushing a92123abacf02fce4f65c3ba94256e83/A: appending metadata at 1731575652084 (+3 ms)Flushing a92123abacf02fce4f65c3ba94256e83/A: closing flushed file at 1731575652084Flushing a92123abacf02fce4f65c3ba94256e83/B: creating writer at 1731575652494 (+410 ms)Flushing a92123abacf02fce4f65c3ba94256e83/B: appending metadata at 1731575652497 (+3 ms)Flushing a92123abacf02fce4f65c3ba94256e83/B: closing flushed file at 1731575652497Flushing a92123abacf02fce4f65c3ba94256e83/C: creating writer at 1731575652907 (+410 ms)Flushing a92123abacf02fce4f65c3ba94256e83/C: appending metadata at 1731575652909 (+2 ms)Flushing a92123abacf02fce4f65c3ba94256e83/C: closing flushed file at 1731575652909Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@289d611: reopening flushed file at 1731575653318 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e751d4: reopening flushed file at 1731575653324 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7f38a9f3: reopening flushed file at 1731575653330 (+6 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for a92123abacf02fce4f65c3ba94256e83 in 1256ms, sequenceid=940, compaction requested=false at 1731575653337 (+7 ms)Writing region close event to WAL at 1731575653485 (+148 ms)Running coprocessor post-close hooks at 1731575653489 (+4 ms)Closed at 1731575653489 2024-11-14T09:14:13,492 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(157): Closed a92123abacf02fce4f65c3ba94256e83 2024-11-14T09:14:13,493 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=a92123abacf02fce4f65c3ba94256e83, regionState=CLOSED 2024-11-14T09:14:13,495 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 because future has completed 2024-11-14T09:14:13,498 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-14T09:14:13,498 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseRegionProcedure a92123abacf02fce4f65c3ba94256e83, server=83f56b55f2af,39927,1731575622393 in 1.5730 sec 2024-11-14T09:14:13,499 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-14T09:14:13,499 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a92123abacf02fce4f65c3ba94256e83, UNASSIGN in 1.5800 sec 2024-11-14T09:14:13,502 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-14T09:14:13,502 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5860 sec 2024-11-14T09:14:13,503 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731575653503"}]},"ts":"1731575653503"} 2024-11-14T09:14:13,504 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T09:14:13,505 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T09:14:13,506 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6010 sec 2024-11-14T09:14:14,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-14T09:14:14,056 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:14:14,059 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-14T09:14:14,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:14:14,065 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:14:14,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-14T09:14:14,066 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=29, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:14:14,071 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83 2024-11-14T09:14:14,074 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A, FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B, FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C, FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/recovered.edits] 2024-11-14T09:14:14,077 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/525a1f337e2546eaa493880a8319619b to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/525a1f337e2546eaa493880a8319619b 2024-11-14T09:14:14,079 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/cd203ee7356a4456b65eb96359750cb1 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/A/cd203ee7356a4456b65eb96359750cb1 2024-11-14T09:14:14,081 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/2f9648d32d224845a175b0244d0cb058 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/2f9648d32d224845a175b0244d0cb058 2024-11-14T09:14:14,083 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/baffaefdb3f244b2bd770626ce331c26 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/B/baffaefdb3f244b2bd770626ce331c26 2024-11-14T09:14:14,085 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/b08b0e2604fb42609a71b247977501c7 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/b08b0e2604fb42609a71b247977501c7 2024-11-14T09:14:14,086 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/c01befaff87e4fb58f9c8c98bce34893 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/C/c01befaff87e4fb58f9c8c98bce34893 2024-11-14T09:14:14,089 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/recovered.edits/943.seqid to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83/recovered.edits/943.seqid 2024-11-14T09:14:14,090 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a92123abacf02fce4f65c3ba94256e83 2024-11-14T09:14:14,090 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T09:14:14,094 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=29, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:14:14,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-14T09:14:14,103 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T09:14:14,135 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T09:14:14,136 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=29, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:14:14,136 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T09:14:14,137 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731575654136"}]},"ts":"9223372036854775807"} 2024-11-14T09:14:14,140 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T09:14:14,140 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a92123abacf02fce4f65c3ba94256e83, NAME => 'TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T09:14:14,140 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T09:14:14,141 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731575654140"}]},"ts":"9223372036854775807"} 2024-11-14T09:14:14,143 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T09:14:14,144 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=29, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:14:14,145 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 84 msec 2024-11-14T09:14:14,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-14T09:14:14,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T09:14:14,176 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:14:14,190 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=248 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_677740128_22 at /127.0.0.1:41976 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;83f56b55f2af:39927-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2015826418_22 at /127.0.0.1:53168 [Waiting for operation #341] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=408 (was 279) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7497 (was 7344) - AvailableMemoryMB LEAK? - 2024-11-14T09:14:14,200 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=408, ProcessCount=11, AvailableMemoryMB=7496 2024-11-14T09:14:14,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T09:14:14,202 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T09:14:14,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T09:14:14,204 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T09:14:14,204 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:14,204 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 30 2024-11-14T09:14:14,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-14T09:14:14,205 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T09:14:14,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741951_1127 (size=1061) 2024-11-14T09:14:14,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-14T09:14:14,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-14T09:14:14,613 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db 2024-11-14T09:14:14,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741952_1128 (size=53) 2024-11-14T09:14:14,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-14T09:14:15,020 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:14:15,021 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing a7663ca41075544279db903393226a2f, disabling compactions & flushes 2024-11-14T09:14:15,021 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:15,021 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:15,021 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. after waiting 0 ms 2024-11-14T09:14:15,021 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:15,021 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:15,021 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for a7663ca41075544279db903393226a2f: Waiting for close lock at 1731575655021Disabling compacts and flushes for region at 1731575655021Disabling writes for close at 1731575655021Writing region close event to WAL at 1731575655021Closed at 1731575655021 2024-11-14T09:14:15,022 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T09:14:15,022 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731575655022"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731575655022"}]},"ts":"1731575655022"} 2024-11-14T09:14:15,024 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T09:14:15,025 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T09:14:15,025 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731575655025"}]},"ts":"1731575655025"} 2024-11-14T09:14:15,026 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T09:14:15,027 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a7663ca41075544279db903393226a2f, ASSIGN}] 2024-11-14T09:14:15,028 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a7663ca41075544279db903393226a2f, ASSIGN 2024-11-14T09:14:15,028 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a7663ca41075544279db903393226a2f, ASSIGN; state=OFFLINE, location=83f56b55f2af,39927,1731575622393; forceNewPlan=false, retain=false 2024-11-14T09:14:15,179 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=a7663ca41075544279db903393226a2f, regionState=OPENING, regionLocation=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:15,181 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a7663ca41075544279db903393226a2f, ASSIGN because future has completed 2024-11-14T09:14:15,182 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393}] 2024-11-14T09:14:15,337 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:15,337 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} 2024-11-14T09:14:15,338 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a7663ca41075544279db903393226a2f 2024-11-14T09:14:15,338 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:14:15,338 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for a7663ca41075544279db903393226a2f 2024-11-14T09:14:15,338 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for a7663ca41075544279db903393226a2f 2024-11-14T09:14:15,339 INFO [StoreOpener-a7663ca41075544279db903393226a2f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a7663ca41075544279db903393226a2f 2024-11-14T09:14:15,340 INFO [StoreOpener-a7663ca41075544279db903393226a2f-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T09:14:15,341 INFO [StoreOpener-a7663ca41075544279db903393226a2f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a7663ca41075544279db903393226a2f columnFamilyName A 2024-11-14T09:14:15,341 DEBUG [StoreOpener-a7663ca41075544279db903393226a2f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:15,342 INFO [StoreOpener-a7663ca41075544279db903393226a2f-1 {}] regionserver.HStore(327): Store=a7663ca41075544279db903393226a2f/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:14:15,342 INFO [StoreOpener-a7663ca41075544279db903393226a2f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a7663ca41075544279db903393226a2f 2024-11-14T09:14:15,343 INFO [StoreOpener-a7663ca41075544279db903393226a2f-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T09:14:15,343 INFO [StoreOpener-a7663ca41075544279db903393226a2f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a7663ca41075544279db903393226a2f columnFamilyName B 2024-11-14T09:14:15,343 DEBUG [StoreOpener-a7663ca41075544279db903393226a2f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:15,344 INFO [StoreOpener-a7663ca41075544279db903393226a2f-1 {}] regionserver.HStore(327): Store=a7663ca41075544279db903393226a2f/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:14:15,344 INFO [StoreOpener-a7663ca41075544279db903393226a2f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a7663ca41075544279db903393226a2f 2024-11-14T09:14:15,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-14T09:14:15,345 INFO [StoreOpener-a7663ca41075544279db903393226a2f-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T09:14:15,345 INFO [StoreOpener-a7663ca41075544279db903393226a2f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a7663ca41075544279db903393226a2f columnFamilyName C 2024-11-14T09:14:15,345 DEBUG [StoreOpener-a7663ca41075544279db903393226a2f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:15,346 INFO [StoreOpener-a7663ca41075544279db903393226a2f-1 {}] regionserver.HStore(327): Store=a7663ca41075544279db903393226a2f/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:14:15,346 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:15,346 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for a7663ca41075544279db903393226a2f 2024-11-14T09:14:15,346 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f 2024-11-14T09:14:15,347 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f 2024-11-14T09:14:15,348 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for a7663ca41075544279db903393226a2f 2024-11-14T09:14:15,348 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for a7663ca41075544279db903393226a2f 2024-11-14T09:14:15,348 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T09:14:15,349 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for a7663ca41075544279db903393226a2f 2024-11-14T09:14:15,351 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T09:14:15,352 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened a7663ca41075544279db903393226a2f; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63934847, jitterRate=-0.047296538949012756}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T09:14:15,352 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a7663ca41075544279db903393226a2f 2024-11-14T09:14:15,353 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for a7663ca41075544279db903393226a2f: Running coprocessor pre-open hook at 1731575655338Writing region info on filesystem at 1731575655338Initializing all the Stores at 1731575655339 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731575655339Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731575655339Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731575655339Cleaning up temporary data from old regions at 1731575655348 (+9 ms)Running coprocessor post-open hooks at 1731575655352 (+4 ms)Region opened successfully at 1731575655353 (+1 ms) 2024-11-14T09:14:15,354 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., pid=32, masterSystemTime=1731575655334 2024-11-14T09:14:15,356 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:15,356 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:15,356 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=a7663ca41075544279db903393226a2f, regionState=OPEN, openSeqNum=2, regionLocation=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:15,358 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 because future has completed 2024-11-14T09:14:15,361 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-14T09:14:15,361 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 in 177 msec 2024-11-14T09:14:15,362 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-14T09:14:15,362 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a7663ca41075544279db903393226a2f, ASSIGN in 334 msec 2024-11-14T09:14:15,363 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T09:14:15,363 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731575655363"}]},"ts":"1731575655363"} 2024-11-14T09:14:15,364 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T09:14:15,365 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T09:14:15,366 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1630 sec 2024-11-14T09:14:16,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-14T09:14:16,355 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:14:16,356 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2697ccde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:16,357 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:14:16,357 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:14:16,358 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:14:16,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:14:16,359 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:14:16,359 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3aed869b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:16,359 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:14:16,359 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:14:16,359 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:14:16,360 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33540, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:14:16,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20fec1a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:16,361 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:14:16,363 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:14:16,363 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:14:16,364 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50654, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:14:16,367 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T09:14:16,367 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 83f56b55f2af,45325,1731575621660 2024-11-14T09:14:16,367 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1929654 2024-11-14T09:14:16,367 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T09:14:16,368 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33542, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T09:14:16,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T09:14:16,374 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T09:14:16,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-14T09:14:16,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741953_1129 (size=1097) 2024-11-14T09:14:16,794 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-14T09:14:16,794 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-14T09:14:16,798 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T09:14:16,804 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a7663ca41075544279db903393226a2f, REOPEN/MOVE}] 2024-11-14T09:14:16,805 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a7663ca41075544279db903393226a2f, REOPEN/MOVE 2024-11-14T09:14:16,805 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=a7663ca41075544279db903393226a2f, regionState=CLOSING, regionLocation=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:16,807 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a7663ca41075544279db903393226a2f, REOPEN/MOVE because future has completed 2024-11-14T09:14:16,807 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T09:14:16,807 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393}] 2024-11-14T09:14:16,960 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(122): Close a7663ca41075544279db903393226a2f 2024-11-14T09:14:16,960 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T09:14:16,960 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1722): Closing a7663ca41075544279db903393226a2f, disabling compactions & flushes 2024-11-14T09:14:16,960 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:16,960 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:16,960 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. after waiting 0 ms 2024-11-14T09:14:16,960 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:16,964 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T09:14:16,965 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:16,965 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1676): Region close journal for a7663ca41075544279db903393226a2f: Waiting for close lock at 1731575656960Running coprocessor pre-close hooks at 1731575656960Disabling compacts and flushes for region at 1731575656960Disabling writes for close at 1731575656960Writing region close event to WAL at 1731575656961 (+1 ms)Running coprocessor post-close hooks at 1731575656965 (+4 ms)Closed at 1731575656965 2024-11-14T09:14:16,965 WARN [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionServer(3299): Not adding moved region record: a7663ca41075544279db903393226a2f to self. 2024-11-14T09:14:16,967 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(157): Closed a7663ca41075544279db903393226a2f 2024-11-14T09:14:16,968 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=a7663ca41075544279db903393226a2f, regionState=CLOSED 2024-11-14T09:14:16,969 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 because future has completed 2024-11-14T09:14:16,971 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-14T09:14:16,971 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 in 163 msec 2024-11-14T09:14:16,972 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a7663ca41075544279db903393226a2f, REOPEN/MOVE; state=CLOSED, location=83f56b55f2af,39927,1731575622393; forceNewPlan=false, retain=true 2024-11-14T09:14:17,123 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=a7663ca41075544279db903393226a2f, regionState=OPENING, regionLocation=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:17,124 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a7663ca41075544279db903393226a2f, REOPEN/MOVE because future has completed 2024-11-14T09:14:17,125 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393}] 2024-11-14T09:14:17,280 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:17,280 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} 2024-11-14T09:14:17,280 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a7663ca41075544279db903393226a2f 2024-11-14T09:14:17,281 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:14:17,281 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for a7663ca41075544279db903393226a2f 2024-11-14T09:14:17,281 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for a7663ca41075544279db903393226a2f 2024-11-14T09:14:17,285 INFO [StoreOpener-a7663ca41075544279db903393226a2f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a7663ca41075544279db903393226a2f 2024-11-14T09:14:17,286 INFO [StoreOpener-a7663ca41075544279db903393226a2f-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T09:14:17,291 INFO [StoreOpener-a7663ca41075544279db903393226a2f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a7663ca41075544279db903393226a2f columnFamilyName A 2024-11-14T09:14:17,292 DEBUG [StoreOpener-a7663ca41075544279db903393226a2f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:17,293 INFO [StoreOpener-a7663ca41075544279db903393226a2f-1 {}] regionserver.HStore(327): Store=a7663ca41075544279db903393226a2f/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:14:17,293 INFO [StoreOpener-a7663ca41075544279db903393226a2f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a7663ca41075544279db903393226a2f 2024-11-14T09:14:17,294 INFO [StoreOpener-a7663ca41075544279db903393226a2f-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T09:14:17,294 INFO [StoreOpener-a7663ca41075544279db903393226a2f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a7663ca41075544279db903393226a2f columnFamilyName B 2024-11-14T09:14:17,294 DEBUG [StoreOpener-a7663ca41075544279db903393226a2f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:17,294 INFO [StoreOpener-a7663ca41075544279db903393226a2f-1 {}] regionserver.HStore(327): Store=a7663ca41075544279db903393226a2f/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:14:17,294 INFO [StoreOpener-a7663ca41075544279db903393226a2f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a7663ca41075544279db903393226a2f 2024-11-14T09:14:17,295 INFO [StoreOpener-a7663ca41075544279db903393226a2f-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T09:14:17,295 INFO [StoreOpener-a7663ca41075544279db903393226a2f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a7663ca41075544279db903393226a2f columnFamilyName C 2024-11-14T09:14:17,295 DEBUG [StoreOpener-a7663ca41075544279db903393226a2f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:17,296 INFO [StoreOpener-a7663ca41075544279db903393226a2f-1 {}] regionserver.HStore(327): Store=a7663ca41075544279db903393226a2f/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:14:17,296 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:17,296 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for a7663ca41075544279db903393226a2f 2024-11-14T09:14:17,296 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f 2024-11-14T09:14:17,297 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f 2024-11-14T09:14:17,298 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for a7663ca41075544279db903393226a2f 2024-11-14T09:14:17,298 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for a7663ca41075544279db903393226a2f 2024-11-14T09:14:17,298 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T09:14:17,299 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for a7663ca41075544279db903393226a2f 2024-11-14T09:14:17,300 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened a7663ca41075544279db903393226a2f; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62820794, jitterRate=-0.06389722228050232}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T09:14:17,300 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a7663ca41075544279db903393226a2f 2024-11-14T09:14:17,301 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for a7663ca41075544279db903393226a2f: Running coprocessor pre-open hook at 1731575657281Writing region info on filesystem at 1731575657281Initializing all the Stores at 1731575657282 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731575657282Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731575657282Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731575657282Cleaning up temporary data from old regions at 1731575657298 (+16 ms)Running coprocessor post-open hooks at 1731575657300 (+2 ms)Region opened successfully at 1731575657301 (+1 ms) 2024-11-14T09:14:17,302 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., pid=37, masterSystemTime=1731575657276 2024-11-14T09:14:17,304 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:17,304 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:17,304 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=a7663ca41075544279db903393226a2f, regionState=OPEN, openSeqNum=5, regionLocation=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:17,307 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 because future has completed 2024-11-14T09:14:17,310 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=35 2024-11-14T09:14:17,310 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 in 183 msec 2024-11-14T09:14:17,312 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-14T09:14:17,312 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a7663ca41075544279db903393226a2f, REOPEN/MOVE in 506 msec 2024-11-14T09:14:17,314 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-14T09:14:17,314 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 516 msec 2024-11-14T09:14:17,317 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 940 msec 2024-11-14T09:14:17,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-14T09:14:17,319 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:14:17,320 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cb008, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:17,320 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:14:17,320 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:14:17,321 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:14:17,321 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:14:17,321 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:14:17,322 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7db910e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:17,322 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:14:17,322 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:14:17,322 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:14:17,323 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33552, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:14:17,323 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e1745a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:17,324 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35409f72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:17,324 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:14:17,324 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:14:17,325 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:14:17,325 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:14:17,325 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:14:17,325 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dbb8645, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:17,325 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:14:17,325 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:14:17,326 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:14:17,326 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33564, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:14:17,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f034f4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:17,328 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7944f9cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:17,328 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:14:17,328 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:14:17,328 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:14:17,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:14:17,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:14:17,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b7fbc2d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:17,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:14:17,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:14:17,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:14:17,330 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33584, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:14:17,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7255b1e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:17,331 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38720fa6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:17,332 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:14:17,332 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:14:17,332 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:14:17,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:14:17,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:14:17,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@652439ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:17,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:14:17,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:14:17,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:14:17,334 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33596, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:14:17,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b232a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:17,335 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c77b203, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:17,335 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:14:17,335 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:14:17,336 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:14:17,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:14:17,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:14:17,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@119d554a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:17,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:14:17,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:14:17,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:14:17,337 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33614, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:14:17,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1419a7fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:17,338 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e3f6eb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:17,339 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:14:17,339 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:14:17,339 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:14:17,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:14:17,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:14:17,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22e7aac5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:17,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:14:17,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:14:17,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:14:17,341 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33632, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:14:17,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56ea4391, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:17,342 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40353459, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:17,342 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:14:17,342 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:14:17,342 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:14:17,342 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:14:17,342 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:14:17,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1882995e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:17,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:14:17,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:14:17,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:14:17,343 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33646, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:14:17,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c8cdbc2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:17,344 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@160383af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:17,344 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:14:17,345 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:14:17,345 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:14:17,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:14:17,346 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:14:17,346 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4938e2ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:17,346 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:14:17,346 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:14:17,346 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:14:17,347 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33676, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:14:17,347 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@366abf8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:17,348 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@123a694c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:17,348 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:14:17,348 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:14:17,349 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:14:17,349 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:14:17,349 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:14:17,349 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37261439, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:17,349 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:14:17,349 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:14:17,350 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:14:17,350 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33698, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:14:17,351 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b8288ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:17,352 DEBUG [Thread-652 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:14:17,352 DEBUG [Thread-656 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:14:17,352 DEBUG [Thread-665 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:14:17,352 DEBUG [Thread-659 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:14:17,352 DEBUG [Thread-661 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:14:17,352 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:14:17,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:14:17,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:14:17,353 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:14:17,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:14:17,353 DEBUG [Thread-648 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:14:17,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-14T09:14:17,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:14:17,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:14:17,353 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:14:17,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:14:17,354 DEBUG [Thread-654 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:14:17,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:14:17,354 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:14:17,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-14T09:14:17,355 DEBUG [Thread-650 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:14:17,355 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:14:17,355 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50676, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:14:17,355 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:14:17,355 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50656, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:14:17,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:14:17,355 DEBUG [Thread-663 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:14:17,356 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50688, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:14:17,356 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:14:17,356 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:14:17,356 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50662, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:14:17,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:14:17,356 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50716, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:14:17,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:14:17,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:14:17,357 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:14:17,357 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50700, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:14:17,358 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50728, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:14:17,358 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:14:17,358 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50720, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:14:17,358 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:14:17,359 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50732, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:14:17,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5] 2024-11-14T09:14:17,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5] 2024-11-14T09:14:17,364 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5] 2024-11-14T09:14:17,364 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5] 2024-11-14T09:14:17,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5] 2024-11-14T09:14:17,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5] 2024-11-14T09:14:17,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5] 2024-11-14T09:14:17,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5] 2024-11-14T09:14:17,378 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a7663ca41075544279db903393226a2f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T09:14:17,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=A 2024-11-14T09:14:17,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:17,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=B 2024-11-14T09:14:17,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:17,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=C 2024-11-14T09:14:17,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:17,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on a7663ca41075544279db903393226a2f 2024-11-14T09:14:17,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5] 2024-11-14T09:14:17,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114ee037bda70d54ac59d52662ade1d4571_a7663ca41075544279db903393226a2f is 50, key is test_row_0/A:col10/1731575657376/Put/seqid=0 2024-11-14T09:14:17,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741954_1130 (size=14594) 2024-11-14T09:14:17,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-14T09:14:17,503 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731575657472, max timestamp=1731575657495Num uniques 859; 2024-11-14T09:14:17,503 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-14T09:14:17,506 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731575657472, max timestamp=1731575657495Num uniques 859; 2024-11-14T09:14:17,506 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-14T09:14:17,507 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731575657472, max timestamp=1731575657495Num uniques 859; 2024-11-14T09:14:17,507 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-14T09:14:17,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-14T09:14:17,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:17,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. as already flushing 2024-11-14T09:14:17,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:17,510 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:17,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:17,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:17,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:17,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1731575717552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:17,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:17,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:17,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:17,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:17,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50700 deadline: 1731575717552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:17,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:17,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:17,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:17,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:17,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50676 deadline: 1731575717565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:17,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:17,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:17,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50728 deadline: 1731575717565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:17,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:17,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:17,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:17,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:17,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:17,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:17,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731575717568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:17,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:17,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:17,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:17,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-14T09:14:17,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:17,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. as already flushing 2024-11-14T09:14:17,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:17,663 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:17,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:17,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:17,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-14T09:14:17,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:17,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50700 deadline: 1731575717675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:17,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:17,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1731575717676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:17,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:17,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:17,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:17,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:17,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:17,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:17,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:17,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50728 deadline: 1731575717686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:17,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:17,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:17,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:17,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:17,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50676 deadline: 1731575717686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:17,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:17,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:17,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:17,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:17,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731575717686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:17,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:17,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:17,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:17,817 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-14T09:14:17,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:17,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. as already flushing 2024-11-14T09:14:17,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:17,818 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:17,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:17,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:17,824 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:17,831 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114ee037bda70d54ac59d52662ade1d4571_a7663ca41075544279db903393226a2f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ee037bda70d54ac59d52662ade1d4571_a7663ca41075544279db903393226a2f 2024-11-14T09:14:17,833 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/c4ac2bcfc6ca4df294f2a8a80a44089c, store: [table=TestAcidGuarantees family=A region=a7663ca41075544279db903393226a2f] 2024-11-14T09:14:17,842 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/c4ac2bcfc6ca4df294f2a8a80a44089c is 175, key is test_row_0/A:col10/1731575657376/Put/seqid=0 2024-11-14T09:14:17,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741955_1131 (size=39549) 2024-11-14T09:14:17,860 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/c4ac2bcfc6ca4df294f2a8a80a44089c 2024-11-14T09:14:17,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:17,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:17,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1731575717887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:17,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50700 deadline: 1731575717886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:17,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:17,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:17,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:17,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:17,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:17,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:17,899 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/1770c2a5a27e45369fd59372e7d4d546 is 50, key is test_row_0/B:col10/1731575657376/Put/seqid=0 2024-11-14T09:14:17,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:17,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50728 deadline: 1731575717896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:17,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:17,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50676 deadline: 1731575717896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:17,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:17,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:17,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:17,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:17,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731575717896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:17,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:17,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:17,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:17,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:17,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:17,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:17,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741956_1132 (size=12001) 2024-11-14T09:14:17,915 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/1770c2a5a27e45369fd59372e7d4d546 2024-11-14T09:14:17,951 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/60c8088514a947e6b377cf63a91933f6 is 50, key is test_row_0/C:col10/1731575657376/Put/seqid=0 2024-11-14T09:14:17,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741957_1133 (size=12001) 2024-11-14T09:14:17,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-14T09:14:17,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:17,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. as already flushing 2024-11-14T09:14:17,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:17,973 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:17,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:17,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:17,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-14T09:14:18,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-14T09:14:18,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:18,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. as already flushing 2024-11-14T09:14:18,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:18,127 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:18,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:18,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:18,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:18,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50700 deadline: 1731575718195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:18,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:18,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:18,197 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:18,197 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:18,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1731575718196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:18,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:18,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:18,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:18,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:18,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50728 deadline: 1731575718215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:18,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:18,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:18,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:18,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:18,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50676 deadline: 1731575718215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:18,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:18,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:18,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:18,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:18,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731575718215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:18,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:18,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:18,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:18,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-14T09:14:18,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:18,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. as already flushing 2024-11-14T09:14:18,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:18,281 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:18,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:18,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:18,364 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/60c8088514a947e6b377cf63a91933f6 2024-11-14T09:14:18,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/c4ac2bcfc6ca4df294f2a8a80a44089c as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/c4ac2bcfc6ca4df294f2a8a80a44089c 2024-11-14T09:14:18,379 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/c4ac2bcfc6ca4df294f2a8a80a44089c, entries=200, sequenceid=14, filesize=38.6 K 2024-11-14T09:14:18,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/1770c2a5a27e45369fd59372e7d4d546 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/1770c2a5a27e45369fd59372e7d4d546 2024-11-14T09:14:18,386 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/1770c2a5a27e45369fd59372e7d4d546, entries=150, sequenceid=14, filesize=11.7 K 2024-11-14T09:14:18,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/60c8088514a947e6b377cf63a91933f6 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/60c8088514a947e6b377cf63a91933f6 2024-11-14T09:14:18,392 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/60c8088514a947e6b377cf63a91933f6, entries=150, sequenceid=14, filesize=11.7 K 2024-11-14T09:14:18,393 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for a7663ca41075544279db903393226a2f in 1015ms, sequenceid=14, compaction requested=false 2024-11-14T09:14:18,393 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-14T09:14:18,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a7663ca41075544279db903393226a2f: 2024-11-14T09:14:18,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-14T09:14:18,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:18,435 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing a7663ca41075544279db903393226a2f 3/3 column families, dataSize=140.89 KB heapSize=390.84 KB 2024-11-14T09:14:18,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=A 2024-11-14T09:14:18,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:14:18,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=B 2024-11-14T09:14:18,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:14:18,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=C 2024-11-14T09:14:18,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:14:18,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114852c6c8b32ec44fdad4e4df2826ec39f_a7663ca41075544279db903393226a2f is 50, key is test_row_0/A:col10/1731575657551/Put/seqid=0 2024-11-14T09:14:18,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741958_1134 (size=12154) 2024-11-14T09:14:18,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:18,469 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114852c6c8b32ec44fdad4e4df2826ec39f_a7663ca41075544279db903393226a2f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114852c6c8b32ec44fdad4e4df2826ec39f_a7663ca41075544279db903393226a2f 2024-11-14T09:14:18,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/3a8aad7792c14ec3bccd14e48f663cc1, store: [table=TestAcidGuarantees family=A region=a7663ca41075544279db903393226a2f] 2024-11-14T09:14:18,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/3a8aad7792c14ec3bccd14e48f663cc1 is 175, key is test_row_0/A:col10/1731575657551/Put/seqid=0 2024-11-14T09:14:18,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741959_1135 (size=30955) 2024-11-14T09:14:18,485 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/3a8aad7792c14ec3bccd14e48f663cc1 2024-11-14T09:14:18,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-14T09:14:18,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/77e4e312e03d4318bd5c2df49bffaf4b is 50, key is test_row_0/B:col10/1731575657551/Put/seqid=0 2024-11-14T09:14:18,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741960_1136 (size=12001) 2024-11-14T09:14:18,538 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/77e4e312e03d4318bd5c2df49bffaf4b 2024-11-14T09:14:18,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/4269b05e852c400d903e3575d7db05d9 is 50, key is test_row_0/C:col10/1731575657551/Put/seqid=0 2024-11-14T09:14:18,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741961_1137 (size=12001) 2024-11-14T09:14:18,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on a7663ca41075544279db903393226a2f 2024-11-14T09:14:18,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. as already flushing 2024-11-14T09:14:18,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:18,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1731575718723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:18,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:18,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:18,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:18,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:18,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50700 deadline: 1731575718724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:18,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:18,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:18,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:18,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:18,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50728 deadline: 1731575718725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:18,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:18,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:18,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:18,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:18,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50676 deadline: 1731575718725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:18,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:18,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:18,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:18,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:18,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731575718726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:18,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:18,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:18,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:18,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:18,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1731575718835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:18,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:18,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:18,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:18,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:18,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50700 deadline: 1731575718835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:18,841 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:18,841 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:18,841 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:18,966 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/4269b05e852c400d903e3575d7db05d9 2024-11-14T09:14:18,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/3a8aad7792c14ec3bccd14e48f663cc1 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/3a8aad7792c14ec3bccd14e48f663cc1 2024-11-14T09:14:18,979 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/3a8aad7792c14ec3bccd14e48f663cc1, entries=150, sequenceid=49, filesize=30.2 K 2024-11-14T09:14:18,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/77e4e312e03d4318bd5c2df49bffaf4b as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/77e4e312e03d4318bd5c2df49bffaf4b 2024-11-14T09:14:18,985 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/77e4e312e03d4318bd5c2df49bffaf4b, entries=150, sequenceid=49, filesize=11.7 K 2024-11-14T09:14:18,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/4269b05e852c400d903e3575d7db05d9 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/4269b05e852c400d903e3575d7db05d9 2024-11-14T09:14:18,993 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/4269b05e852c400d903e3575d7db05d9, entries=150, sequenceid=49, filesize=11.7 K 2024-11-14T09:14:18,995 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~390.80 KB/400176, currentSize=46.96 KB/48090 for a7663ca41075544279db903393226a2f in 559ms, sequenceid=49, compaction requested=false 2024-11-14T09:14:18,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for a7663ca41075544279db903393226a2f: 2024-11-14T09:14:18,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:18,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-14T09:14:18,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-14T09:14:19,000 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-14T09:14:19,000 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6410 sec 2024-11-14T09:14:19,002 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 1.6490 sec 2024-11-14T09:14:19,033 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T09:14:19,050 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a7663ca41075544279db903393226a2f 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-14T09:14:19,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=A 2024-11-14T09:14:19,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:19,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=B 2024-11-14T09:14:19,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:19,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=C 2024-11-14T09:14:19,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:19,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on a7663ca41075544279db903393226a2f 2024-11-14T09:14:19,076 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411140ac0442cbec64187820bfa31435e73df_a7663ca41075544279db903393226a2f is 50, key is test_row_0/A:col10/1731575659049/Put/seqid=0 2024-11-14T09:14:19,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741962_1138 (size=17034) 2024-11-14T09:14:19,106 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:19,113 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411140ac0442cbec64187820bfa31435e73df_a7663ca41075544279db903393226a2f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411140ac0442cbec64187820bfa31435e73df_a7663ca41075544279db903393226a2f 2024-11-14T09:14:19,114 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/d5935a348e3a4260a785f5c2be1a2683, store: [table=TestAcidGuarantees family=A region=a7663ca41075544279db903393226a2f] 2024-11-14T09:14:19,115 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/d5935a348e3a4260a785f5c2be1a2683 is 175, key is test_row_0/A:col10/1731575659049/Put/seqid=0 2024-11-14T09:14:19,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:19,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50700 deadline: 1731575719113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:19,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:19,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:19,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:19,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741963_1139 (size=48139) 2024-11-14T09:14:19,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:19,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1731575719123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:19,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:19,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:19,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:19,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:19,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50700 deadline: 1731575719225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:19,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:19,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:19,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:19,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:19,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1731575719235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:19,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:19,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:19,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:19,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:19,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50700 deadline: 1731575719435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:19,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:19,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:19,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:19,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:19,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1731575719445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:19,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:19,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:19,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:19,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-14T09:14:19,506 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:14:19,506 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:14:19,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-14T09:14:19,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T09:14:19,508 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:14:19,509 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:14:19,509 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:14:19,527 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=61, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/d5935a348e3a4260a785f5c2be1a2683 2024-11-14T09:14:19,542 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/841223c0058a41479b01cc749482c576 is 50, key is test_row_0/B:col10/1731575659049/Put/seqid=0 2024-11-14T09:14:19,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741964_1140 (size=12001) 2024-11-14T09:14:19,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T09:14:19,662 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T09:14:19,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:19,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. as already flushing 2024-11-14T09:14:19,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:19,662 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:19,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:19,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:19,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:19,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50728 deadline: 1731575719745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:19,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:19,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50700 deadline: 1731575719746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:19,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:19,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:19,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50676 deadline: 1731575719746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:19,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:19,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:19,747 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_1' in a7663ca41075544279db903393226a2f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:14:19,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:19,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:19,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:19,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:19,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731575719746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:19,748 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_2' in a7663ca41075544279db903393226a2f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:14:19,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:19,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:19,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:19,749 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_2' in a7663ca41075544279db903393226a2f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:14:19,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:19,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:19,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:19,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:19,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1731575719765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:19,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:19,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:19,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:19,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T09:14:19,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:19,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. as already flushing 2024-11-14T09:14:19,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:19,817 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:19,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:19,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:19,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T09:14:19,952 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/841223c0058a41479b01cc749482c576 2024-11-14T09:14:19,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/2bf56dff9e3c4ad0ae2e440129be874e is 50, key is test_row_0/C:col10/1731575659049/Put/seqid=0 2024-11-14T09:14:19,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741965_1141 (size=12001) 2024-11-14T09:14:19,967 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/2bf56dff9e3c4ad0ae2e440129be874e 2024-11-14T09:14:19,970 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T09:14:19,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:19,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. as already flushing 2024-11-14T09:14:19,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:19,971 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:19,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:19,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/d5935a348e3a4260a785f5c2be1a2683 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/d5935a348e3a4260a785f5c2be1a2683 2024-11-14T09:14:19,978 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/d5935a348e3a4260a785f5c2be1a2683, entries=250, sequenceid=61, filesize=47.0 K 2024-11-14T09:14:19,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/841223c0058a41479b01cc749482c576 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/841223c0058a41479b01cc749482c576 2024-11-14T09:14:19,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:19,986 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/841223c0058a41479b01cc749482c576, entries=150, sequenceid=61, filesize=11.7 K 2024-11-14T09:14:19,988 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/2bf56dff9e3c4ad0ae2e440129be874e as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/2bf56dff9e3c4ad0ae2e440129be874e 2024-11-14T09:14:19,992 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/2bf56dff9e3c4ad0ae2e440129be874e, entries=150, sequenceid=61, filesize=11.7 K 2024-11-14T09:14:19,993 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for a7663ca41075544279db903393226a2f in 943ms, sequenceid=61, compaction requested=true 2024-11-14T09:14:19,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a7663ca41075544279db903393226a2f: 2024-11-14T09:14:19,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7663ca41075544279db903393226a2f:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:14:19,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:19,994 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:14:19,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7663ca41075544279db903393226a2f:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:14:19,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:19,994 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:14:19,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7663ca41075544279db903393226a2f:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:14:19,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:14:19,996 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:14:19,996 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:14:19,996 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): a7663ca41075544279db903393226a2f/B is initiating minor compaction (all files) 2024-11-14T09:14:19,996 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): a7663ca41075544279db903393226a2f/A is initiating minor compaction (all files) 2024-11-14T09:14:19,996 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7663ca41075544279db903393226a2f/A in TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:19,996 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7663ca41075544279db903393226a2f/B in TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:19,997 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/1770c2a5a27e45369fd59372e7d4d546, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/77e4e312e03d4318bd5c2df49bffaf4b, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/841223c0058a41479b01cc749482c576] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp, totalSize=35.2 K 2024-11-14T09:14:19,997 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/c4ac2bcfc6ca4df294f2a8a80a44089c, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/3a8aad7792c14ec3bccd14e48f663cc1, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/d5935a348e3a4260a785f5c2be1a2683] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp, totalSize=115.9 K 2024-11-14T09:14:19,997 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:19,997 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. files: [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/c4ac2bcfc6ca4df294f2a8a80a44089c, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/3a8aad7792c14ec3bccd14e48f663cc1, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/d5935a348e3a4260a785f5c2be1a2683] 2024-11-14T09:14:19,997 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 1770c2a5a27e45369fd59372e7d4d546, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731575657369 2024-11-14T09:14:19,997 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4ac2bcfc6ca4df294f2a8a80a44089c, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731575657369 2024-11-14T09:14:19,998 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 77e4e312e03d4318bd5c2df49bffaf4b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731575657551 2024-11-14T09:14:19,998 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a8aad7792c14ec3bccd14e48f663cc1, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731575657551 2024-11-14T09:14:19,998 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 841223c0058a41479b01cc749482c576, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731575658715 2024-11-14T09:14:19,999 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5935a348e3a4260a785f5c2be1a2683, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731575658715 2024-11-14T09:14:20,010 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a7663ca41075544279db903393226a2f] 2024-11-14T09:14:20,012 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7663ca41075544279db903393226a2f#B#compaction#123 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:14:20,013 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/20edc1fd08ea483aaa89d21139a23229 is 50, key is test_row_0/B:col10/1731575659049/Put/seqid=0 2024-11-14T09:14:20,016 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114f37a9809b00346b5b6e6540357a347a2_a7663ca41075544279db903393226a2f store=[table=TestAcidGuarantees family=A region=a7663ca41075544279db903393226a2f] 2024-11-14T09:14:20,024 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114f37a9809b00346b5b6e6540357a347a2_a7663ca41075544279db903393226a2f, store=[table=TestAcidGuarantees family=A region=a7663ca41075544279db903393226a2f] 2024-11-14T09:14:20,024 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114f37a9809b00346b5b6e6540357a347a2_a7663ca41075544279db903393226a2f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a7663ca41075544279db903393226a2f] 2024-11-14T09:14:20,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741966_1142 (size=12104) 2024-11-14T09:14:20,043 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/20edc1fd08ea483aaa89d21139a23229 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/20edc1fd08ea483aaa89d21139a23229 2024-11-14T09:14:20,053 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7663ca41075544279db903393226a2f/B of a7663ca41075544279db903393226a2f into 20edc1fd08ea483aaa89d21139a23229(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:14:20,053 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7663ca41075544279db903393226a2f: 2024-11-14T09:14:20,053 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., storeName=a7663ca41075544279db903393226a2f/B, priority=13, startTime=1731575659994; duration=0sec 2024-11-14T09:14:20,053 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:14:20,053 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7663ca41075544279db903393226a2f:B 2024-11-14T09:14:20,053 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:14:20,054 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:14:20,055 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): a7663ca41075544279db903393226a2f/C is initiating minor compaction (all files) 2024-11-14T09:14:20,055 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7663ca41075544279db903393226a2f/C in TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:20,055 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/60c8088514a947e6b377cf63a91933f6, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/4269b05e852c400d903e3575d7db05d9, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/2bf56dff9e3c4ad0ae2e440129be874e] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp, totalSize=35.2 K 2024-11-14T09:14:20,055 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 60c8088514a947e6b377cf63a91933f6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731575657369 2024-11-14T09:14:20,056 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 4269b05e852c400d903e3575d7db05d9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731575657551 2024-11-14T09:14:20,057 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bf56dff9e3c4ad0ae2e440129be874e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731575658715 2024-11-14T09:14:20,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741967_1143 (size=4469) 2024-11-14T09:14:20,072 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7663ca41075544279db903393226a2f#A#compaction#124 average throughput is 0.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:14:20,072 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7663ca41075544279db903393226a2f#C#compaction#125 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:14:20,073 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/257504f0297f42acb267366a3479c519 is 175, key is test_row_0/A:col10/1731575659049/Put/seqid=0 2024-11-14T09:14:20,073 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/d4553efcb62048789668c031a3879b08 is 50, key is test_row_0/C:col10/1731575659049/Put/seqid=0 2024-11-14T09:14:20,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741968_1144 (size=31058) 2024-11-14T09:14:20,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741969_1145 (size=12104) 2024-11-14T09:14:20,105 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/d4553efcb62048789668c031a3879b08 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/d4553efcb62048789668c031a3879b08 2024-11-14T09:14:20,115 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7663ca41075544279db903393226a2f/C of a7663ca41075544279db903393226a2f into d4553efcb62048789668c031a3879b08(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:14:20,115 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7663ca41075544279db903393226a2f: 2024-11-14T09:14:20,115 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., storeName=a7663ca41075544279db903393226a2f/C, priority=13, startTime=1731575659994; duration=0sec 2024-11-14T09:14:20,115 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:20,115 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7663ca41075544279db903393226a2f:C 2024-11-14T09:14:20,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T09:14:20,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:20,132 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing a7663ca41075544279db903393226a2f 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-14T09:14:20,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=A 2024-11-14T09:14:20,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:20,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=B 2024-11-14T09:14:20,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:20,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=C 2024-11-14T09:14:20,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:20,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T09:14:20,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111492544ea31f894bbc8464da2ca81a29f3_a7663ca41075544279db903393226a2f is 50, key is test_row_0/A:col10/1731575659108/Put/seqid=0 2024-11-14T09:14:20,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741970_1146 (size=12154) 2024-11-14T09:14:20,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:20,170 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111492544ea31f894bbc8464da2ca81a29f3_a7663ca41075544279db903393226a2f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111492544ea31f894bbc8464da2ca81a29f3_a7663ca41075544279db903393226a2f 2024-11-14T09:14:20,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/f99360a3e1144084aaf8eddae09b1e18, store: [table=TestAcidGuarantees family=A region=a7663ca41075544279db903393226a2f] 2024-11-14T09:14:20,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/f99360a3e1144084aaf8eddae09b1e18 is 175, key is test_row_0/A:col10/1731575659108/Put/seqid=0 2024-11-14T09:14:20,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741971_1147 (size=30955) 2024-11-14T09:14:20,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on a7663ca41075544279db903393226a2f 2024-11-14T09:14:20,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. as already flushing 2024-11-14T09:14:20,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:20,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1731575720278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:20,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:20,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:20,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:20,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:20,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50700 deadline: 1731575720281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:20,288 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:20,288 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:20,288 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:20,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:20,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1731575720384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:20,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:20,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:20,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:20,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:20,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50700 deadline: 1731575720395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:20,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:20,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:20,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:20,499 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/257504f0297f42acb267366a3479c519 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/257504f0297f42acb267366a3479c519 2024-11-14T09:14:20,505 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7663ca41075544279db903393226a2f/A of a7663ca41075544279db903393226a2f into 257504f0297f42acb267366a3479c519(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:14:20,505 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7663ca41075544279db903393226a2f: 2024-11-14T09:14:20,505 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., storeName=a7663ca41075544279db903393226a2f/A, priority=13, startTime=1731575659993; duration=0sec 2024-11-14T09:14:20,505 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:20,505 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7663ca41075544279db903393226a2f:A 2024-11-14T09:14:20,595 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=83, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/f99360a3e1144084aaf8eddae09b1e18 2024-11-14T09:14:20,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:20,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1731575720594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:20,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:20,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:20,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:20,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/d2e6394e8c6d48cd90969bb270098478 is 50, key is test_row_0/B:col10/1731575659108/Put/seqid=0 2024-11-14T09:14:20,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:20,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50700 deadline: 1731575720605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:20,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:20,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741972_1148 (size=12001) 2024-11-14T09:14:20,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:20,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:20,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T09:14:20,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:20,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1731575720905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:20,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:20,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:20,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:20,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:20,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50700 deadline: 1731575720924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:20,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:20,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:20,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:21,010 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/d2e6394e8c6d48cd90969bb270098478 2024-11-14T09:14:21,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/88d919a94517449aa85c96885501043b is 50, key is test_row_0/C:col10/1731575659108/Put/seqid=0 2024-11-14T09:14:21,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741973_1149 (size=12001) 2024-11-14T09:14:21,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:21,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1731575721415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:21,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:21,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:21,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:21,428 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/88d919a94517449aa85c96885501043b 2024-11-14T09:14:21,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/f99360a3e1144084aaf8eddae09b1e18 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/f99360a3e1144084aaf8eddae09b1e18 2024-11-14T09:14:21,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:21,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50700 deadline: 1731575721434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:21,437 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:21,437 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:21,437 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:21,441 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/f99360a3e1144084aaf8eddae09b1e18, entries=150, sequenceid=83, filesize=30.2 K 2024-11-14T09:14:21,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/d2e6394e8c6d48cd90969bb270098478 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/d2e6394e8c6d48cd90969bb270098478 2024-11-14T09:14:21,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,448 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/d2e6394e8c6d48cd90969bb270098478, entries=150, sequenceid=83, filesize=11.7 K 2024-11-14T09:14:21,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/88d919a94517449aa85c96885501043b as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/88d919a94517449aa85c96885501043b 2024-11-14T09:14:21,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,456 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/88d919a94517449aa85c96885501043b, entries=150, sequenceid=83, filesize=11.7 K 2024-11-14T09:14:21,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,458 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for a7663ca41075544279db903393226a2f in 1326ms, sequenceid=83, compaction requested=false 2024-11-14T09:14:21,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for a7663ca41075544279db903393226a2f: 2024-11-14T09:14:21,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:21,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-14T09:14:21,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-14T09:14:21,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,462 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-14T09:14:21,463 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9510 sec 2024-11-14T09:14:21,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,465 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.9570 sec 2024-11-14T09:14:21,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T09:14:21,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,655 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:14:21,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,656 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:14:21,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-14T09:14:21,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T09:14:21,675 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:14:21,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,676 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:14:21,676 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:14:21,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T09:14:21,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on a7663ca41075544279db903393226a2f 2024-11-14T09:14:21,800 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a7663ca41075544279db903393226a2f 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-14T09:14:21,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=A 2024-11-14T09:14:21,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:21,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=B 2024-11-14T09:14:21,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:21,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=C 2024-11-14T09:14:21,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:21,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,813 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e0af3287ff7b40aba2a32bae31c80b82_a7663ca41075544279db903393226a2f is 50, key is test_row_1/A:col10/1731575661797/Put/seqid=0 2024-11-14T09:14:21,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741974_1150 (size=14594) 2024-11-14T09:14:21,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T09:14:21,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:21,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. as already flushing 2024-11-14T09:14:21,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:21,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,830 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:21,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:21,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:21,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,833 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,839 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e0af3287ff7b40aba2a32bae31c80b82_a7663ca41075544279db903393226a2f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e0af3287ff7b40aba2a32bae31c80b82_a7663ca41075544279db903393226a2f 2024-11-14T09:14:21,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,840 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/b790b7f0e7f44c9a9eed181d3270f72f, store: [table=TestAcidGuarantees family=A region=a7663ca41075544279db903393226a2f] 2024-11-14T09:14:21,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,841 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/b790b7f0e7f44c9a9eed181d3270f72f is 175, key is test_row_1/A:col10/1731575661797/Put/seqid=0 2024-11-14T09:14:21,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:21,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741975_1151 (size=39545) 2024-11-14T09:14:21,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:21,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50676 deadline: 1731575721882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:21,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:21,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:21,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:21,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:21,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50728 deadline: 1731575721883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:21,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:21,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:21,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731575721884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:21,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:21,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:21,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:21,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:21,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:21,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T09:14:21,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:21,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. as already flushing 2024-11-14T09:14:21,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:21,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:21,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:21,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:21,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:21,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50676 deadline: 1731575721995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:21,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:21,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50728 deadline: 1731575721995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:21,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T09:14:21,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:21,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:21,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:21,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731575721996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:21,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:22,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:22,000 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:22,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:22,000 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:22,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:22,000 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:22,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T09:14:22,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:22,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. as already flushing 2024-11-14T09:14:22,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:22,138 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:22,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:22,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:22,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:22,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50728 deadline: 1731575722205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:22,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:22,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:22,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731575722205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:22,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:22,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:22,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:22,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:22,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:22,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:22,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50676 deadline: 1731575722205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:22,208 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:22,208 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:22,208 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:22,256 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=99, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/b790b7f0e7f44c9a9eed181d3270f72f 2024-11-14T09:14:22,264 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/abc4b3a7e05846b5a232533daf6eac49 is 50, key is test_row_1/B:col10/1731575661797/Put/seqid=0 2024-11-14T09:14:22,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741976_1152 (size=9657) 2024-11-14T09:14:22,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T09:14:22,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:22,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. as already flushing 2024-11-14T09:14:22,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:22,291 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:22,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:22,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:22,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T09:14:22,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:22,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1731575722425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:22,427 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_1' in a7663ca41075544279db903393226a2f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2149 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:14:22,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:22,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:22,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:22,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T09:14:22,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:22,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. as already flushing 2024-11-14T09:14:22,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:22,445 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:22,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:22,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:22,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:22,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50700 deadline: 1731575722444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:22,447 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_1' in a7663ca41075544279db903393226a2f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:14:22,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:22,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:22,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:22,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:22,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:22,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50728 deadline: 1731575722515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:22,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731575722515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:22,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:22,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:22,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:22,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50676 deadline: 1731575722515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:22,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:22,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:22,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:22,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:22,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:22,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:22,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:22,599 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T09:14:22,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:22,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. as already flushing 2024-11-14T09:14:22,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:22,599 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:22,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:22,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:22,670 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/abc4b3a7e05846b5a232533daf6eac49 2024-11-14T09:14:22,679 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/914e6f83fd684c6398cc2af0acbb8e87 is 50, key is test_row_1/C:col10/1731575661797/Put/seqid=0 2024-11-14T09:14:22,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741977_1153 (size=9657) 2024-11-14T09:14:22,752 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T09:14:22,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:22,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. as already flushing 2024-11-14T09:14:22,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:22,753 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:22,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:22,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:22,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T09:14:22,906 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T09:14:22,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:22,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. as already flushing 2024-11-14T09:14:22,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:22,906 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:22,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:22,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:23,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:23,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50728 deadline: 1731575723025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:23,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:23,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:23,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:23,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:23,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50676 deadline: 1731575723025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:23,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:23,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:23,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:23,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:23,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731575723025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:23,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:23,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:23,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:23,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T09:14:23,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:23,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. as already flushing 2024-11-14T09:14:23,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:23,060 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:23,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:23,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:23,087 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/914e6f83fd684c6398cc2af0acbb8e87 2024-11-14T09:14:23,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/b790b7f0e7f44c9a9eed181d3270f72f as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/b790b7f0e7f44c9a9eed181d3270f72f 2024-11-14T09:14:23,105 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/b790b7f0e7f44c9a9eed181d3270f72f, entries=200, sequenceid=99, filesize=38.6 K 2024-11-14T09:14:23,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/abc4b3a7e05846b5a232533daf6eac49 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/abc4b3a7e05846b5a232533daf6eac49 2024-11-14T09:14:23,116 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/abc4b3a7e05846b5a232533daf6eac49, entries=100, sequenceid=99, filesize=9.4 K 2024-11-14T09:14:23,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/914e6f83fd684c6398cc2af0acbb8e87 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/914e6f83fd684c6398cc2af0acbb8e87 2024-11-14T09:14:23,133 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/914e6f83fd684c6398cc2af0acbb8e87, entries=100, sequenceid=99, filesize=9.4 K 2024-11-14T09:14:23,134 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for a7663ca41075544279db903393226a2f in 1334ms, sequenceid=99, compaction requested=true 2024-11-14T09:14:23,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a7663ca41075544279db903393226a2f: 2024-11-14T09:14:23,134 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:14:23,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7663ca41075544279db903393226a2f:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:14:23,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:23,135 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:14:23,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7663ca41075544279db903393226a2f:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:14:23,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:23,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7663ca41075544279db903393226a2f:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:14:23,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:14:23,145 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101558 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:14:23,145 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): a7663ca41075544279db903393226a2f/A is initiating minor compaction (all files) 2024-11-14T09:14:23,145 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7663ca41075544279db903393226a2f/A in TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:23,145 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/257504f0297f42acb267366a3479c519, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/f99360a3e1144084aaf8eddae09b1e18, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/b790b7f0e7f44c9a9eed181d3270f72f] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp, totalSize=99.2 K 2024-11-14T09:14:23,145 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:23,145 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. files: [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/257504f0297f42acb267366a3479c519, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/f99360a3e1144084aaf8eddae09b1e18, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/b790b7f0e7f44c9a9eed181d3270f72f] 2024-11-14T09:14:23,146 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:14:23,146 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): a7663ca41075544279db903393226a2f/B is initiating minor compaction (all files) 2024-11-14T09:14:23,146 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7663ca41075544279db903393226a2f/B in TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:23,146 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 257504f0297f42acb267366a3479c519, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731575658715 2024-11-14T09:14:23,146 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/20edc1fd08ea483aaa89d21139a23229, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/d2e6394e8c6d48cd90969bb270098478, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/abc4b3a7e05846b5a232533daf6eac49] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp, totalSize=33.0 K 2024-11-14T09:14:23,146 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting f99360a3e1144084aaf8eddae09b1e18, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731575659108 2024-11-14T09:14:23,146 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20edc1fd08ea483aaa89d21139a23229, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731575658715 2024-11-14T09:14:23,147 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting b790b7f0e7f44c9a9eed181d3270f72f, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731575660276 2024-11-14T09:14:23,147 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2e6394e8c6d48cd90969bb270098478, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731575659108 2024-11-14T09:14:23,148 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting abc4b3a7e05846b5a232533daf6eac49, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731575661797 2024-11-14T09:14:23,157 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a7663ca41075544279db903393226a2f] 2024-11-14T09:14:23,159 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7663ca41075544279db903393226a2f#B#compaction#133 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:14:23,160 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/012130478ed144d9a4486c7542fd1f14 is 50, key is test_row_0/B:col10/1731575659108/Put/seqid=0 2024-11-14T09:14:23,165 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111455120b343b6d47528902e4d84c2daaa7_a7663ca41075544279db903393226a2f store=[table=TestAcidGuarantees family=A region=a7663ca41075544279db903393226a2f] 2024-11-14T09:14:23,167 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111455120b343b6d47528902e4d84c2daaa7_a7663ca41075544279db903393226a2f, store=[table=TestAcidGuarantees family=A region=a7663ca41075544279db903393226a2f] 2024-11-14T09:14:23,168 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111455120b343b6d47528902e4d84c2daaa7_a7663ca41075544279db903393226a2f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a7663ca41075544279db903393226a2f] 2024-11-14T09:14:23,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741978_1154 (size=12207) 2024-11-14T09:14:23,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741979_1155 (size=4469) 2024-11-14T09:14:23,179 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/012130478ed144d9a4486c7542fd1f14 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/012130478ed144d9a4486c7542fd1f14 2024-11-14T09:14:23,181 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7663ca41075544279db903393226a2f#A#compaction#132 average throughput is 1.02 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:14:23,183 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/89c3b595f2ee49b98d9bf4c7c47afaa0 is 175, key is test_row_0/A:col10/1731575659108/Put/seqid=0 2024-11-14T09:14:23,188 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7663ca41075544279db903393226a2f/B of a7663ca41075544279db903393226a2f into 012130478ed144d9a4486c7542fd1f14(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:14:23,188 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7663ca41075544279db903393226a2f: 2024-11-14T09:14:23,188 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., storeName=a7663ca41075544279db903393226a2f/B, priority=13, startTime=1731575663135; duration=0sec 2024-11-14T09:14:23,189 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:14:23,189 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7663ca41075544279db903393226a2f:B 2024-11-14T09:14:23,189 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:14:23,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741980_1156 (size=31268) 2024-11-14T09:14:23,201 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:14:23,201 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): a7663ca41075544279db903393226a2f/C is initiating minor compaction (all files) 2024-11-14T09:14:23,201 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7663ca41075544279db903393226a2f/C in TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:23,201 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/d4553efcb62048789668c031a3879b08, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/88d919a94517449aa85c96885501043b, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/914e6f83fd684c6398cc2af0acbb8e87] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp, totalSize=33.0 K 2024-11-14T09:14:23,202 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4553efcb62048789668c031a3879b08, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731575658715 2024-11-14T09:14:23,203 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88d919a94517449aa85c96885501043b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731575659108 2024-11-14T09:14:23,204 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 914e6f83fd684c6398cc2af0acbb8e87, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731575661797 2024-11-14T09:14:23,213 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/89c3b595f2ee49b98d9bf4c7c47afaa0 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/89c3b595f2ee49b98d9bf4c7c47afaa0 2024-11-14T09:14:23,213 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T09:14:23,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:23,214 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing a7663ca41075544279db903393226a2f 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-14T09:14:23,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=A 2024-11-14T09:14:23,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:23,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=B 2024-11-14T09:14:23,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:23,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=C 2024-11-14T09:14:23,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:23,227 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7663ca41075544279db903393226a2f/A of a7663ca41075544279db903393226a2f into 89c3b595f2ee49b98d9bf4c7c47afaa0(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:14:23,227 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7663ca41075544279db903393226a2f: 2024-11-14T09:14:23,227 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., storeName=a7663ca41075544279db903393226a2f/A, priority=13, startTime=1731575663134; duration=0sec 2024-11-14T09:14:23,227 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:23,227 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7663ca41075544279db903393226a2f:A 2024-11-14T09:14:23,231 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7663ca41075544279db903393226a2f#C#compaction#134 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:14:23,231 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/c484909f18e44f7d828ae2327e8fbcda is 50, key is test_row_0/C:col10/1731575659108/Put/seqid=0 2024-11-14T09:14:23,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111487210816b2564dabb7520fabd5b85cf4_a7663ca41075544279db903393226a2f is 50, key is test_row_0/A:col10/1731575661883/Put/seqid=0 2024-11-14T09:14:23,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741981_1157 (size=12207) 2024-11-14T09:14:23,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741982_1158 (size=12154) 2024-11-14T09:14:23,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,646 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/c484909f18e44f7d828ae2327e8fbcda as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/c484909f18e44f7d828ae2327e8fbcda 2024-11-14T09:14:23,646 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111487210816b2564dabb7520fabd5b85cf4_a7663ca41075544279db903393226a2f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111487210816b2564dabb7520fabd5b85cf4_a7663ca41075544279db903393226a2f 2024-11-14T09:14:23,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/8a97df37932547978448d8d7f9ea9ca3, store: [table=TestAcidGuarantees family=A region=a7663ca41075544279db903393226a2f] 2024-11-14T09:14:23,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/8a97df37932547978448d8d7f9ea9ca3 is 175, key is test_row_0/A:col10/1731575661883/Put/seqid=0 2024-11-14T09:14:23,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741983_1159 (size=30955) 2024-11-14T09:14:23,658 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7663ca41075544279db903393226a2f/C of a7663ca41075544279db903393226a2f into c484909f18e44f7d828ae2327e8fbcda(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:14:23,658 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7663ca41075544279db903393226a2f: 2024-11-14T09:14:23,658 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., storeName=a7663ca41075544279db903393226a2f/C, priority=13, startTime=1731575663135; duration=0sec 2024-11-14T09:14:23,659 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/8a97df37932547978448d8d7f9ea9ca3 2024-11-14T09:14:23,660 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:23,660 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7663ca41075544279db903393226a2f:C 2024-11-14T09:14:23,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/3355906649184b49885c06eb320bc551 is 50, key is test_row_0/B:col10/1731575661883/Put/seqid=0 2024-11-14T09:14:23,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741984_1160 (size=12001) 2024-11-14T09:14:23,681 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/3355906649184b49885c06eb320bc551 2024-11-14T09:14:23,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/7b3b47e54feb4f458c271ddfb1f7056a is 50, key is test_row_0/C:col10/1731575661883/Put/seqid=0 2024-11-14T09:14:23,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741985_1161 (size=12001) 2024-11-14T09:14:23,746 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/7b3b47e54feb4f458c271ddfb1f7056a 2024-11-14T09:14:23,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/8a97df37932547978448d8d7f9ea9ca3 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/8a97df37932547978448d8d7f9ea9ca3 2024-11-14T09:14:23,762 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/8a97df37932547978448d8d7f9ea9ca3, entries=150, sequenceid=117, filesize=30.2 K 2024-11-14T09:14:23,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/3355906649184b49885c06eb320bc551 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/3355906649184b49885c06eb320bc551 2024-11-14T09:14:23,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,768 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/3355906649184b49885c06eb320bc551, entries=150, sequenceid=117, filesize=11.7 K 2024-11-14T09:14:23,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/7b3b47e54feb4f458c271ddfb1f7056a as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/7b3b47e54feb4f458c271ddfb1f7056a 2024-11-14T09:14:23,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,785 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/7b3b47e54feb4f458c271ddfb1f7056a, entries=150, sequenceid=117, filesize=11.7 K 2024-11-14T09:14:23,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,787 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=0 B/0 for a7663ca41075544279db903393226a2f in 573ms, sequenceid=117, compaction requested=false 2024-11-14T09:14:23,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for a7663ca41075544279db903393226a2f: 2024-11-14T09:14:23,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:23,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-14T09:14:23,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-14T09:14:23,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,791 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-14T09:14:23,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,791 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1130 sec 2024-11-14T09:14:23,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,794 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.1360 sec 2024-11-14T09:14:23,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T09:14:23,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,835 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:14:23,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,836 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:14:23,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-14T09:14:23,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,839 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:14:23,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,840 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:14:23,841 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:14:23,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T09:14:23,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T09:14:23,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-14T09:14:23,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:23,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for a7663ca41075544279db903393226a2f: 2024-11-14T09:14:23,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:23,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-14T09:14:23,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-14T09:14:23,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,998 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-14T09:14:23,998 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 155 msec 2024-11-14T09:14:23,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:23,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,001 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 163 msec 2024-11-14T09:14:24,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on a7663ca41075544279db903393226a2f 2024-11-14T09:14:24,100 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a7663ca41075544279db903393226a2f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T09:14:24,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=A 2024-11-14T09:14:24,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:24,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=B 2024-11-14T09:14:24,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:24,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=C 2024-11-14T09:14:24,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:24,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,111 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114853416af207c41ffb5d2340c811c05f1_a7663ca41075544279db903393226a2f is 50, key is test_row_0/A:col10/1731575664093/Put/seqid=0 2024-11-14T09:14:24,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T09:14:24,156 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:14:24,157 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:14:24,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-14T09:14:24,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T09:14:24,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741986_1162 (size=22018) 2024-11-14T09:14:24,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,160 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:14:24,160 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,161 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:14:24,161 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:14:24,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,166 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114853416af207c41ffb5d2340c811c05f1_a7663ca41075544279db903393226a2f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114853416af207c41ffb5d2340c811c05f1_a7663ca41075544279db903393226a2f 2024-11-14T09:14:24,167 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/a71691244e514baabdc0f4f5c74b0383, store: [table=TestAcidGuarantees family=A region=a7663ca41075544279db903393226a2f] 2024-11-14T09:14:24,168 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/a71691244e514baabdc0f4f5c74b0383 is 175, key is test_row_0/A:col10/1731575664093/Put/seqid=0 2024-11-14T09:14:24,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741987_1163 (size=65423) 2024-11-14T09:14:24,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,187 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=130, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/a71691244e514baabdc0f4f5c74b0383 2024-11-14T09:14:24,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1731575664101, max timestamp=1731575664185Num uniques 859; 2024-11-14T09:14:24,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-11-14T09:14:24,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,188 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1731575664101, max timestamp=1731575664185Num uniques 859; 2024-11-14T09:14:24,188 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-11-14T09:14:24,189 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1731575664101, max timestamp=1731575664185Num uniques 859; 2024-11-14T09:14:24,189 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-11-14T09:14:24,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:24,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731575724197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:24,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:24,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:24,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50728 deadline: 1731575724198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:24,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:24,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:24,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:24,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:24,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:24,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50676 deadline: 1731575724198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:24,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:24,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:24,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:24,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:24,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/90f19a64cbc245c7b151c712018b6ee6 is 50, key is test_row_0/B:col10/1731575664093/Put/seqid=0 2024-11-14T09:14:24,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741988_1164 (size=21465) 2024-11-14T09:14:24,241 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/90f19a64cbc245c7b151c712018b6ee6 2024-11-14T09:14:24,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/3b6196f58d984aa6aac52c90969fdf26 is 50, key is test_row_0/C:col10/1731575664093/Put/seqid=0 2024-11-14T09:14:24,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T09:14:24,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:24,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731575724305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:24,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:24,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50728 deadline: 1731575724305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:24,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:24,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:24,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:24,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:24,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:24,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:24,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T09:14:24,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:24,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. as already flushing 2024-11-14T09:14:24,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:24,316 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:24,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:24,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:24,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:24,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50676 deadline: 1731575724316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:24,324 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:24,324 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:24,324 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:24,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741989_1165 (size=12101) 2024-11-14T09:14:24,325 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/3b6196f58d984aa6aac52c90969fdf26 2024-11-14T09:14:24,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/a71691244e514baabdc0f4f5c74b0383 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/a71691244e514baabdc0f4f5c74b0383 2024-11-14T09:14:24,339 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/a71691244e514baabdc0f4f5c74b0383, entries=350, sequenceid=130, filesize=63.9 K 2024-11-14T09:14:24,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/90f19a64cbc245c7b151c712018b6ee6 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/90f19a64cbc245c7b151c712018b6ee6 2024-11-14T09:14:24,352 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/90f19a64cbc245c7b151c712018b6ee6, entries=350, sequenceid=130, filesize=21.0 K 2024-11-14T09:14:24,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/3b6196f58d984aa6aac52c90969fdf26 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/3b6196f58d984aa6aac52c90969fdf26 2024-11-14T09:14:24,360 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/3b6196f58d984aa6aac52c90969fdf26, entries=150, sequenceid=130, filesize=11.8 K 2024-11-14T09:14:24,361 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=154.31 KB/158010 for a7663ca41075544279db903393226a2f in 261ms, sequenceid=130, compaction requested=true 2024-11-14T09:14:24,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a7663ca41075544279db903393226a2f: 2024-11-14T09:14:24,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7663ca41075544279db903393226a2f:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:14:24,361 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:14:24,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:24,361 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:14:24,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7663ca41075544279db903393226a2f:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:14:24,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:24,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7663ca41075544279db903393226a2f:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:14:24,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:14:24,363 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127646 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:14:24,365 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45673 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:14:24,365 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): a7663ca41075544279db903393226a2f/B is initiating minor compaction (all files) 2024-11-14T09:14:24,365 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7663ca41075544279db903393226a2f/B in TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:24,365 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/012130478ed144d9a4486c7542fd1f14, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/3355906649184b49885c06eb320bc551, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/90f19a64cbc245c7b151c712018b6ee6] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp, totalSize=44.6 K 2024-11-14T09:14:24,365 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): a7663ca41075544279db903393226a2f/A is initiating minor compaction (all files) 2024-11-14T09:14:24,365 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7663ca41075544279db903393226a2f/A in TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:24,365 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/89c3b595f2ee49b98d9bf4c7c47afaa0, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/8a97df37932547978448d8d7f9ea9ca3, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/a71691244e514baabdc0f4f5c74b0383] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp, totalSize=124.7 K 2024-11-14T09:14:24,365 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:24,366 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. files: [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/89c3b595f2ee49b98d9bf4c7c47afaa0, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/8a97df37932547978448d8d7f9ea9ca3, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/a71691244e514baabdc0f4f5c74b0383] 2024-11-14T09:14:24,367 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89c3b595f2ee49b98d9bf4c7c47afaa0, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731575659108 2024-11-14T09:14:24,367 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 012130478ed144d9a4486c7542fd1f14, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731575659108 2024-11-14T09:14:24,367 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a97df37932547978448d8d7f9ea9ca3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731575661864 2024-11-14T09:14:24,368 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 3355906649184b49885c06eb320bc551, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731575661864 2024-11-14T09:14:24,368 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting a71691244e514baabdc0f4f5c74b0383, keycount=350, bloomtype=ROW, size=63.9 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731575664059 2024-11-14T09:14:24,369 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 90f19a64cbc245c7b151c712018b6ee6, keycount=350, bloomtype=ROW, size=21.0 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731575664059 2024-11-14T09:14:24,395 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7663ca41075544279db903393226a2f#B#compaction#141 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:14:24,396 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/5183162510e34f2aa19ddb7335381967 is 50, key is test_row_0/B:col10/1731575664093/Put/seqid=0 2024-11-14T09:14:24,403 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a7663ca41075544279db903393226a2f] 2024-11-14T09:14:24,416 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114ff1af8815ab14a4e99df26295bbfbd22_a7663ca41075544279db903393226a2f store=[table=TestAcidGuarantees family=A region=a7663ca41075544279db903393226a2f] 2024-11-14T09:14:24,419 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114ff1af8815ab14a4e99df26295bbfbd22_a7663ca41075544279db903393226a2f, store=[table=TestAcidGuarantees family=A region=a7663ca41075544279db903393226a2f] 2024-11-14T09:14:24,419 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114ff1af8815ab14a4e99df26295bbfbd22_a7663ca41075544279db903393226a2f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a7663ca41075544279db903393226a2f] 2024-11-14T09:14:24,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741990_1166 (size=12409) 2024-11-14T09:14:24,432 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/5183162510e34f2aa19ddb7335381967 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/5183162510e34f2aa19ddb7335381967 2024-11-14T09:14:24,440 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a7663ca41075544279db903393226a2f 3/3 column families, dataSize=161.02 KB heapSize=399.05 KB 2024-11-14T09:14:24,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on a7663ca41075544279db903393226a2f 2024-11-14T09:14:24,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=A 2024-11-14T09:14:24,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:14:24,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=B 2024-11-14T09:14:24,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:14:24,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=C 2024-11-14T09:14:24,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:14:24,443 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7663ca41075544279db903393226a2f/B of a7663ca41075544279db903393226a2f into 5183162510e34f2aa19ddb7335381967(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:14:24,443 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7663ca41075544279db903393226a2f: 2024-11-14T09:14:24,443 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., storeName=a7663ca41075544279db903393226a2f/B, priority=13, startTime=1731575664361; duration=0sec 2024-11-14T09:14:24,443 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:14:24,443 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7663ca41075544279db903393226a2f:B 2024-11-14T09:14:24,443 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:14:24,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741991_1167 (size=4469) 2024-11-14T09:14:24,447 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:14:24,447 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): a7663ca41075544279db903393226a2f/C is initiating minor compaction (all files) 2024-11-14T09:14:24,447 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7663ca41075544279db903393226a2f/C in TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:24,448 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/c484909f18e44f7d828ae2327e8fbcda, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/7b3b47e54feb4f458c271ddfb1f7056a, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/3b6196f58d984aa6aac52c90969fdf26] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp, totalSize=35.5 K 2024-11-14T09:14:24,449 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting c484909f18e44f7d828ae2327e8fbcda, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731575659108 2024-11-14T09:14:24,449 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b3b47e54feb4f458c271ddfb1f7056a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731575661864 2024-11-14T09:14:24,449 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7663ca41075544279db903393226a2f#A#compaction#142 average throughput is 0.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:14:24,450 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b6196f58d984aa6aac52c90969fdf26, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731575664061 2024-11-14T09:14:24,450 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/748399112a2a4a62849e60e2faadb109 is 175, key is test_row_0/A:col10/1731575664093/Put/seqid=0 2024-11-14T09:14:24,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T09:14:24,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:24,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. as already flushing 2024-11-14T09:14:24,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:24,469 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:24,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:24,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:24,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T09:14:24,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114a557f8b7b383465a86a26b70bdc438d9_a7663ca41075544279db903393226a2f is 50, key is test_row_0/A:col10/1731575664196/Put/seqid=0 2024-11-14T09:14:24,493 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7663ca41075544279db903393226a2f#C#compaction#144 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:14:24,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:24,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1731575724483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:24,494 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/27357202a8b94a9698da80b0447a8f93 is 50, key is test_row_0/C:col10/1731575664093/Put/seqid=0 2024-11-14T09:14:24,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:24,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:24,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:24,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:24,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50700 deadline: 1731575724485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:24,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:24,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:24,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:24,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741992_1168 (size=31363) 2024-11-14T09:14:24,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741993_1169 (size=14794) 2024-11-14T09:14:24,517 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:24,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:24,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50728 deadline: 1731575724515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:24,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:24,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:24,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:24,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:24,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731575724516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:24,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:24,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:24,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:24,522 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/748399112a2a4a62849e60e2faadb109 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/748399112a2a4a62849e60e2faadb109 2024-11-14T09:14:24,522 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114a557f8b7b383465a86a26b70bdc438d9_a7663ca41075544279db903393226a2f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114a557f8b7b383465a86a26b70bdc438d9_a7663ca41075544279db903393226a2f 2024-11-14T09:14:24,530 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/5d9bb5c1df7e482c84cc6b16f9d4b289, store: [table=TestAcidGuarantees family=A region=a7663ca41075544279db903393226a2f] 2024-11-14T09:14:24,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/5d9bb5c1df7e482c84cc6b16f9d4b289 is 175, key is test_row_0/A:col10/1731575664196/Put/seqid=0 2024-11-14T09:14:24,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:24,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50676 deadline: 1731575724535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:24,537 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:24,537 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:24,537 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:24,540 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7663ca41075544279db903393226a2f/A of a7663ca41075544279db903393226a2f into 748399112a2a4a62849e60e2faadb109(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:14:24,540 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7663ca41075544279db903393226a2f: 2024-11-14T09:14:24,540 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., storeName=a7663ca41075544279db903393226a2f/A, priority=13, startTime=1731575664361; duration=0sec 2024-11-14T09:14:24,540 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:24,540 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7663ca41075544279db903393226a2f:A 2024-11-14T09:14:24,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741994_1170 (size=12409) 2024-11-14T09:14:24,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741995_1171 (size=39749) 2024-11-14T09:14:24,566 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=157, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/5d9bb5c1df7e482c84cc6b16f9d4b289 2024-11-14T09:14:24,577 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/5626061d07c9440c9f7d4ca0e5a5428b is 50, key is test_row_0/B:col10/1731575664196/Put/seqid=0 2024-11-14T09:14:24,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741996_1172 (size=12151) 2024-11-14T09:14:24,594 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/5626061d07c9440c9f7d4ca0e5a5428b 2024-11-14T09:14:24,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/36b5fd906bee437fb49684c3df86b592 is 50, key is test_row_0/C:col10/1731575664196/Put/seqid=0 2024-11-14T09:14:24,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:24,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1731575724605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:24,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:24,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:24,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:24,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:24,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50700 deadline: 1731575724605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:24,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:24,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:24,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:24,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741997_1173 (size=12151) 2024-11-14T09:14:24,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T09:14:24,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:24,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. as already flushing 2024-11-14T09:14:24,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:24,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:24,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:24,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:24,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T09:14:24,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:24,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. as already flushing 2024-11-14T09:14:24,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:24,778 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:24,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:24,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:24,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T09:14:24,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:24,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1731575724815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:24,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:24,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50700 deadline: 1731575724815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:24,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:24,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:24,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:24,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:24,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50728 deadline: 1731575724825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:24,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:24,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:24,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:24,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:24,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:24,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:24,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:24,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731575724826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:24,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:24,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:24,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:24,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:24,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50676 deadline: 1731575724845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:24,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:24,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:24,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:24,932 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T09:14:24,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:24,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. as already flushing 2024-11-14T09:14:24,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:24,932 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:24,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:24,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:24,955 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/27357202a8b94a9698da80b0447a8f93 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/27357202a8b94a9698da80b0447a8f93 2024-11-14T09:14:24,963 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7663ca41075544279db903393226a2f/C of a7663ca41075544279db903393226a2f into 27357202a8b94a9698da80b0447a8f93(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:14:24,963 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7663ca41075544279db903393226a2f: 2024-11-14T09:14:24,963 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., storeName=a7663ca41075544279db903393226a2f/C, priority=13, startTime=1731575664361; duration=0sec 2024-11-14T09:14:24,964 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:24,965 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7663ca41075544279db903393226a2f:C 2024-11-14T09:14:25,023 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/36b5fd906bee437fb49684c3df86b592 2024-11-14T09:14:25,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/5d9bb5c1df7e482c84cc6b16f9d4b289 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/5d9bb5c1df7e482c84cc6b16f9d4b289 2024-11-14T09:14:25,071 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/5d9bb5c1df7e482c84cc6b16f9d4b289, entries=200, sequenceid=157, filesize=38.8 K 2024-11-14T09:14:25,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/5626061d07c9440c9f7d4ca0e5a5428b as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/5626061d07c9440c9f7d4ca0e5a5428b 2024-11-14T09:14:25,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/5626061d07c9440c9f7d4ca0e5a5428b, entries=150, sequenceid=157, filesize=11.9 K 2024-11-14T09:14:25,087 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/36b5fd906bee437fb49684c3df86b592 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/36b5fd906bee437fb49684c3df86b592 2024-11-14T09:14:25,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T09:14:25,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:25,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. as already flushing 2024-11-14T09:14:25,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:25,092 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:25,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:25,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:25,097 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/36b5fd906bee437fb49684c3df86b592, entries=150, sequenceid=157, filesize=11.9 K 2024-11-14T09:14:25,098 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~399 KB/408576, currentSize=40.25 KB/41220 for a7663ca41075544279db903393226a2f in 658ms, sequenceid=157, compaction requested=false 2024-11-14T09:14:25,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a7663ca41075544279db903393226a2f: 2024-11-14T09:14:25,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on a7663ca41075544279db903393226a2f 2024-11-14T09:14:25,138 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a7663ca41075544279db903393226a2f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T09:14:25,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=A 2024-11-14T09:14:25,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:25,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=B 2024-11-14T09:14:25,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:25,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=C 2024-11-14T09:14:25,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:25,158 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111420999d04577d438085dbc17c3f46e678_a7663ca41075544279db903393226a2f is 50, key is test_row_0/A:col10/1731575664472/Put/seqid=0 2024-11-14T09:14:25,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741998_1174 (size=17284) 2024-11-14T09:14:25,174 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:25,181 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111420999d04577d438085dbc17c3f46e678_a7663ca41075544279db903393226a2f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111420999d04577d438085dbc17c3f46e678_a7663ca41075544279db903393226a2f 2024-11-14T09:14:25,183 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/d567524433ef49ff9c84264d0811d8cf, store: [table=TestAcidGuarantees family=A region=a7663ca41075544279db903393226a2f] 2024-11-14T09:14:25,184 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/d567524433ef49ff9c84264d0811d8cf is 175, key is test_row_0/A:col10/1731575664472/Put/seqid=0 2024-11-14T09:14:25,193 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731575665161, max timestamp=1731575665189Num uniques 859; 2024-11-14T09:14:25,193 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-14T09:14:25,194 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731575665161, max timestamp=1731575665189Num uniques 859; 2024-11-14T09:14:25,194 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-14T09:14:25,194 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731575665161, max timestamp=1731575665189Num uniques 859; 2024-11-14T09:14:25,194 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-14T09:14:25,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741999_1175 (size=48389) 2024-11-14T09:14:25,229 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=171, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/d567524433ef49ff9c84264d0811d8cf 2024-11-14T09:14:25,241 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/e40ada958b0b467d8b0d98d7223c4e4b is 50, key is test_row_0/B:col10/1731575664472/Put/seqid=0 2024-11-14T09:14:25,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T09:14:25,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:25,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. as already flushing 2024-11-14T09:14:25,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:25,245 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:25,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:25,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:25,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:25,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1731575725243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:25,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:25,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50700 deadline: 1731575725244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:25,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:25,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:25,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:25,251 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:25,251 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:25,251 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:25,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742000_1176 (size=12151) 2024-11-14T09:14:25,254 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/e40ada958b0b467d8b0d98d7223c4e4b 2024-11-14T09:14:25,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/7ada07d0b2e74a46863798f7a2074a41 is 50, key is test_row_0/C:col10/1731575664472/Put/seqid=0 2024-11-14T09:14:25,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T09:14:25,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742001_1177 (size=12151) 2024-11-14T09:14:25,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:25,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50728 deadline: 1731575725335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:25,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:25,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:25,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:25,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:25,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731575725335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:25,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:25,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:25,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:25,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:25,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50700 deadline: 1731575725355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:25,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:25,360 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:25,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1731575725355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:25,360 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:25,360 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:25,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:25,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:25,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:25,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:25,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50676 deadline: 1731575725365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:25,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:25,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:25,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:25,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T09:14:25,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:25,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. as already flushing 2024-11-14T09:14:25,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:25,399 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:25,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:25,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:25,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T09:14:25,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:25,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. as already flushing 2024-11-14T09:14:25,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:25,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:25,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:25,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:25,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:25,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50700 deadline: 1731575725565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:25,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:25,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:25,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:25,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1731575725565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:25,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:25,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:25,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:25,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:25,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T09:14:25,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:25,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. as already flushing 2024-11-14T09:14:25,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:25,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:25,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:25,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:25,710 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/7ada07d0b2e74a46863798f7a2074a41 2024-11-14T09:14:25,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/d567524433ef49ff9c84264d0811d8cf as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/d567524433ef49ff9c84264d0811d8cf 2024-11-14T09:14:25,721 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/d567524433ef49ff9c84264d0811d8cf, entries=250, sequenceid=171, filesize=47.3 K 2024-11-14T09:14:25,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/e40ada958b0b467d8b0d98d7223c4e4b as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/e40ada958b0b467d8b0d98d7223c4e4b 2024-11-14T09:14:25,728 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/e40ada958b0b467d8b0d98d7223c4e4b, entries=150, sequenceid=171, filesize=11.9 K 2024-11-14T09:14:25,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/7ada07d0b2e74a46863798f7a2074a41 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/7ada07d0b2e74a46863798f7a2074a41 2024-11-14T09:14:25,736 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/7ada07d0b2e74a46863798f7a2074a41, entries=150, sequenceid=171, filesize=11.9 K 2024-11-14T09:14:25,737 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for a7663ca41075544279db903393226a2f in 599ms, sequenceid=171, compaction requested=true 2024-11-14T09:14:25,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a7663ca41075544279db903393226a2f: 2024-11-14T09:14:25,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7663ca41075544279db903393226a2f:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:14:25,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:25,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7663ca41075544279db903393226a2f:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:14:25,738 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:14:25,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:14:25,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7663ca41075544279db903393226a2f:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:14:25,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T09:14:25,738 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:14:25,739 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 119501 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:14:25,739 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): a7663ca41075544279db903393226a2f/A is initiating minor compaction (all files) 2024-11-14T09:14:25,739 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7663ca41075544279db903393226a2f/A in TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:25,740 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/748399112a2a4a62849e60e2faadb109, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/5d9bb5c1df7e482c84cc6b16f9d4b289, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/d567524433ef49ff9c84264d0811d8cf] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp, totalSize=116.7 K 2024-11-14T09:14:25,740 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:25,740 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. files: [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/748399112a2a4a62849e60e2faadb109, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/5d9bb5c1df7e482c84cc6b16f9d4b289, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/d567524433ef49ff9c84264d0811d8cf] 2024-11-14T09:14:25,740 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:14:25,740 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 748399112a2a4a62849e60e2faadb109, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731575664061 2024-11-14T09:14:25,740 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): a7663ca41075544279db903393226a2f/B is initiating minor compaction (all files) 2024-11-14T09:14:25,740 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7663ca41075544279db903393226a2f/B in TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:25,741 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/5183162510e34f2aa19ddb7335381967, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/5626061d07c9440c9f7d4ca0e5a5428b, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/e40ada958b0b467d8b0d98d7223c4e4b] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp, totalSize=35.9 K 2024-11-14T09:14:25,741 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d9bb5c1df7e482c84cc6b16f9d4b289, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731575664185 2024-11-14T09:14:25,741 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 5183162510e34f2aa19ddb7335381967, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731575664061 2024-11-14T09:14:25,741 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting d567524433ef49ff9c84264d0811d8cf, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731575664472 2024-11-14T09:14:25,742 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 5626061d07c9440c9f7d4ca0e5a5428b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731575664185 2024-11-14T09:14:25,742 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting e40ada958b0b467d8b0d98d7223c4e4b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731575664472 2024-11-14T09:14:25,760 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7663ca41075544279db903393226a2f#B#compaction#150 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:14:25,761 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/6210f261404d457bba5b21ad46ac3aa9 is 50, key is test_row_0/B:col10/1731575664472/Put/seqid=0 2024-11-14T09:14:25,769 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a7663ca41075544279db903393226a2f] 2024-11-14T09:14:25,790 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114e8c4c49014e24907894944ee7d2846a2_a7663ca41075544279db903393226a2f store=[table=TestAcidGuarantees family=A region=a7663ca41075544279db903393226a2f] 2024-11-14T09:14:25,792 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114e8c4c49014e24907894944ee7d2846a2_a7663ca41075544279db903393226a2f, store=[table=TestAcidGuarantees family=A region=a7663ca41075544279db903393226a2f] 2024-11-14T09:14:25,793 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e8c4c49014e24907894944ee7d2846a2_a7663ca41075544279db903393226a2f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a7663ca41075544279db903393226a2f] 2024-11-14T09:14:25,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742002_1178 (size=12561) 2024-11-14T09:14:25,838 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/6210f261404d457bba5b21ad46ac3aa9 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/6210f261404d457bba5b21ad46ac3aa9 2024-11-14T09:14:25,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742003_1179 (size=4469) 2024-11-14T09:14:25,844 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7663ca41075544279db903393226a2f#A#compaction#151 average throughput is 0.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:14:25,845 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7663ca41075544279db903393226a2f/B of a7663ca41075544279db903393226a2f into 6210f261404d457bba5b21ad46ac3aa9(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:14:25,845 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7663ca41075544279db903393226a2f: 2024-11-14T09:14:25,845 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., storeName=a7663ca41075544279db903393226a2f/B, priority=13, startTime=1731575665738; duration=0sec 2024-11-14T09:14:25,845 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:14:25,845 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7663ca41075544279db903393226a2f:B 2024-11-14T09:14:25,845 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/d0b8fc44900d438dafe9b526ef2971f3 is 175, key is test_row_0/A:col10/1731575664472/Put/seqid=0 2024-11-14T09:14:25,845 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:14:25,848 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:14:25,848 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): a7663ca41075544279db903393226a2f/C is initiating minor compaction (all files) 2024-11-14T09:14:25,848 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7663ca41075544279db903393226a2f/C in TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:25,848 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/27357202a8b94a9698da80b0447a8f93, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/36b5fd906bee437fb49684c3df86b592, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/7ada07d0b2e74a46863798f7a2074a41] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp, totalSize=35.9 K 2024-11-14T09:14:25,849 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 27357202a8b94a9698da80b0447a8f93, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731575664061 2024-11-14T09:14:25,849 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 36b5fd906bee437fb49684c3df86b592, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731575664185 2024-11-14T09:14:25,849 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ada07d0b2e74a46863798f7a2074a41, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731575664472 2024-11-14T09:14:25,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T09:14:25,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:25,862 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing a7663ca41075544279db903393226a2f 3/3 column families, dataSize=134.18 KB heapSize=366.23 KB 2024-11-14T09:14:25,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=A 2024-11-14T09:14:25,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:14:25,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=B 2024-11-14T09:14:25,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:14:25,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=C 2024-11-14T09:14:25,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:14:25,877 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7663ca41075544279db903393226a2f#C#compaction#152 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:14:25,878 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/b08275aba6c34e949583c53876920b5f is 50, key is test_row_0/C:col10/1731575664472/Put/seqid=0 2024-11-14T09:14:25,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. as already flushing 2024-11-14T09:14:25,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on a7663ca41075544279db903393226a2f 2024-11-14T09:14:25,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742004_1180 (size=31515) 2024-11-14T09:14:25,900 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/d0b8fc44900d438dafe9b526ef2971f3 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/d0b8fc44900d438dafe9b526ef2971f3 2024-11-14T09:14:25,907 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7663ca41075544279db903393226a2f/A of a7663ca41075544279db903393226a2f into d0b8fc44900d438dafe9b526ef2971f3(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:14:25,907 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7663ca41075544279db903393226a2f: 2024-11-14T09:14:25,907 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., storeName=a7663ca41075544279db903393226a2f/A, priority=13, startTime=1731575665738; duration=0sec 2024-11-14T09:14:25,907 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:25,907 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7663ca41075544279db903393226a2f:A 2024-11-14T09:14:25,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114c322142bfe994cad805b26bf4c47eb17_a7663ca41075544279db903393226a2f is 50, key is test_row_0/A:col10/1731575665224/Put/seqid=0 2024-11-14T09:14:25,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:25,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50700 deadline: 1731575725906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:25,910 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:25,910 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:25,910 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:25,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742005_1181 (size=12561) 2024-11-14T09:14:25,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:25,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1731575725910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:25,920 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:25,920 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:25,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:25,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742006_1182 (size=12304) 2024-11-14T09:14:25,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:25,962 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114c322142bfe994cad805b26bf4c47eb17_a7663ca41075544279db903393226a2f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c322142bfe994cad805b26bf4c47eb17_a7663ca41075544279db903393226a2f 2024-11-14T09:14:25,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/24dd8082057a4581a1fbd40574a574f2, store: [table=TestAcidGuarantees family=A region=a7663ca41075544279db903393226a2f] 2024-11-14T09:14:25,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/24dd8082057a4581a1fbd40574a574f2 is 175, key is test_row_0/A:col10/1731575665224/Put/seqid=0 2024-11-14T09:14:25,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742007_1183 (size=31105) 2024-11-14T09:14:26,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:26,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50700 deadline: 1731575726015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:26,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:26,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:26,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:26,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:26,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1731575726025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:26,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:26,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:26,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:26,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:26,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50700 deadline: 1731575726225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:26,232 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:26,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:26,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:26,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:26,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1731575726234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:26,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:26,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:26,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:26,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T09:14:26,327 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/b08275aba6c34e949583c53876920b5f as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/b08275aba6c34e949583c53876920b5f 2024-11-14T09:14:26,330 INFO [master/83f56b55f2af:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-14T09:14:26,330 INFO [master/83f56b55f2af:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-14T09:14:26,340 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7663ca41075544279db903393226a2f/C of a7663ca41075544279db903393226a2f into b08275aba6c34e949583c53876920b5f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:14:26,340 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7663ca41075544279db903393226a2f: 2024-11-14T09:14:26,340 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., storeName=a7663ca41075544279db903393226a2f/C, priority=13, startTime=1731575665738; duration=0sec 2024-11-14T09:14:26,340 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:26,340 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7663ca41075544279db903393226a2f:C 2024-11-14T09:14:26,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:26,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50728 deadline: 1731575726355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:26,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:26,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731575726355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:26,361 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_0' in a7663ca41075544279db903393226a2f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:14:26,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:26,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:26,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:26,361 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_2' in a7663ca41075544279db903393226a2f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:14:26,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:26,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:26,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:26,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:26,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50676 deadline: 1731575726387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:26,388 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_0' in a7663ca41075544279db903393226a2f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:14:26,389 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:26,389 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:26,389 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:26,390 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=205, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/24dd8082057a4581a1fbd40574a574f2 2024-11-14T09:14:26,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/0bcd4c2f6f6f45578fc7eb40b25de789 is 50, key is test_row_0/B:col10/1731575665224/Put/seqid=0 2024-11-14T09:14:26,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742008_1184 (size=12151) 2024-11-14T09:14:26,423 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/0bcd4c2f6f6f45578fc7eb40b25de789 2024-11-14T09:14:26,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/a289daf5fdd746e4a4026d539b81b8bf is 50, key is test_row_0/C:col10/1731575665224/Put/seqid=0 2024-11-14T09:14:26,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742009_1185 (size=12151) 2024-11-14T09:14:26,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:26,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50700 deadline: 1731575726545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:26,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:26,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:26,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:26,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1731575726545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:26,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:26,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:26,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:26,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:26,847 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/a289daf5fdd746e4a4026d539b81b8bf 2024-11-14T09:14:26,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/24dd8082057a4581a1fbd40574a574f2 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/24dd8082057a4581a1fbd40574a574f2 2024-11-14T09:14:26,858 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/24dd8082057a4581a1fbd40574a574f2, entries=150, sequenceid=205, filesize=30.4 K 2024-11-14T09:14:26,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/0bcd4c2f6f6f45578fc7eb40b25de789 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/0bcd4c2f6f6f45578fc7eb40b25de789 2024-11-14T09:14:26,864 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/0bcd4c2f6f6f45578fc7eb40b25de789, entries=150, sequenceid=205, filesize=11.9 K 2024-11-14T09:14:26,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/a289daf5fdd746e4a4026d539b81b8bf as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/a289daf5fdd746e4a4026d539b81b8bf 2024-11-14T09:14:26,872 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/a289daf5fdd746e4a4026d539b81b8bf, entries=150, sequenceid=205, filesize=11.9 K 2024-11-14T09:14:26,873 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~366.19 KB/374976, currentSize=53.67 KB/54960 for a7663ca41075544279db903393226a2f in 1012ms, sequenceid=205, compaction requested=false 2024-11-14T09:14:26,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for a7663ca41075544279db903393226a2f: 2024-11-14T09:14:26,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:26,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-14T09:14:26,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-14T09:14:26,881 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-14T09:14:26,881 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7160 sec 2024-11-14T09:14:26,883 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.7250 sec 2024-11-14T09:14:27,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on a7663ca41075544279db903393226a2f 2024-11-14T09:14:27,058 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a7663ca41075544279db903393226a2f 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-14T09:14:27,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=A 2024-11-14T09:14:27,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:27,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=B 2024-11-14T09:14:27,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:27,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=C 2024-11-14T09:14:27,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:27,079 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114f37fcbf495024614a12d3ce86b2e9a7c_a7663ca41075544279db903393226a2f is 50, key is test_row_0/A:col10/1731575665895/Put/seqid=0 2024-11-14T09:14:27,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742010_1186 (size=17284) 2024-11-14T09:14:27,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:27,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1731575727107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:27,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:27,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:27,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50700 deadline: 1731575727109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:27,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:27,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:27,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:27,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:27,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:27,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:27,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1731575727225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:27,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:27,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:27,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:27,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50700 deadline: 1731575727225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:27,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:27,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:27,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:27,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:27,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:27,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1731575727435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:27,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:27,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:27,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:27,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:27,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50700 deadline: 1731575727435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:27,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:27,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:27,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:27,503 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:27,508 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114f37fcbf495024614a12d3ce86b2e9a7c_a7663ca41075544279db903393226a2f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f37fcbf495024614a12d3ce86b2e9a7c_a7663ca41075544279db903393226a2f 2024-11-14T09:14:27,513 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/7d7554db47c0437aafc12ef7aa6970ad, store: [table=TestAcidGuarantees family=A region=a7663ca41075544279db903393226a2f] 2024-11-14T09:14:27,513 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/7d7554db47c0437aafc12ef7aa6970ad is 175, key is test_row_0/A:col10/1731575665895/Put/seqid=0 2024-11-14T09:14:27,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742011_1187 (size=48389) 2024-11-14T09:14:27,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:27,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1731575727745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:27,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:27,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:27,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:27,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:27,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50700 deadline: 1731575727745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:27,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:27,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:27,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:27,928 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=220, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/7d7554db47c0437aafc12ef7aa6970ad 2024-11-14T09:14:27,937 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/0f7f48ff16c64443af965a9110966736 is 50, key is test_row_0/B:col10/1731575665895/Put/seqid=0 2024-11-14T09:14:27,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742012_1188 (size=12151) 2024-11-14T09:14:28,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:28,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50700 deadline: 1731575728255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:28,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:28,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:28,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:28,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:28,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1731575728255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:28,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:28,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:28,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:28,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T09:14:28,335 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:14:28,336 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:14:28,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-14T09:14:28,340 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:14:28,341 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:14:28,341 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:14:28,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-14T09:14:28,356 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/0f7f48ff16c64443af965a9110966736 2024-11-14T09:14:28,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:28,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50728 deadline: 1731575728375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:28,379 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_0' in a7663ca41075544279db903393226a2f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:14:28,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:28,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:28,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:28,381 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/452cfb5cbfa24b23ba032263a1d7c630 is 50, key is test_row_0/C:col10/1731575665895/Put/seqid=0 2024-11-14T09:14:28,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:28,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731575728387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:28,390 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_2' in a7663ca41075544279db903393226a2f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:14:28,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:28,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:28,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:28,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742013_1189 (size=12151) 2024-11-14T09:14:28,404 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/452cfb5cbfa24b23ba032263a1d7c630 2024-11-14T09:14:28,411 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/7d7554db47c0437aafc12ef7aa6970ad as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/7d7554db47c0437aafc12ef7aa6970ad 2024-11-14T09:14:28,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:28,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50676 deadline: 1731575728405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:28,412 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_0' in a7663ca41075544279db903393226a2f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:14:28,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:28,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:28,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:28,433 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/7d7554db47c0437aafc12ef7aa6970ad, entries=250, sequenceid=220, filesize=47.3 K 2024-11-14T09:14:28,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/0f7f48ff16c64443af965a9110966736 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/0f7f48ff16c64443af965a9110966736 2024-11-14T09:14:28,440 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/0f7f48ff16c64443af965a9110966736, entries=150, sequenceid=220, filesize=11.9 K 2024-11-14T09:14:28,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/452cfb5cbfa24b23ba032263a1d7c630 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/452cfb5cbfa24b23ba032263a1d7c630 2024-11-14T09:14:28,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-14T09:14:28,448 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/452cfb5cbfa24b23ba032263a1d7c630, entries=150, sequenceid=220, filesize=11.9 K 2024-11-14T09:14:28,449 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for a7663ca41075544279db903393226a2f in 1391ms, sequenceid=220, compaction requested=true 2024-11-14T09:14:28,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a7663ca41075544279db903393226a2f: 2024-11-14T09:14:28,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7663ca41075544279db903393226a2f:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:14:28,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:28,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7663ca41075544279db903393226a2f:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:14:28,450 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:14:28,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:28,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7663ca41075544279db903393226a2f:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:14:28,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:14:28,450 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:14:28,451 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111009 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:14:28,451 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): a7663ca41075544279db903393226a2f/A is initiating minor compaction (all files) 2024-11-14T09:14:28,451 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7663ca41075544279db903393226a2f/A in TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:28,451 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:14:28,451 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/d0b8fc44900d438dafe9b526ef2971f3, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/24dd8082057a4581a1fbd40574a574f2, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/7d7554db47c0437aafc12ef7aa6970ad] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp, totalSize=108.4 K 2024-11-14T09:14:28,451 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): a7663ca41075544279db903393226a2f/B is initiating minor compaction (all files) 2024-11-14T09:14:28,451 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:28,452 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7663ca41075544279db903393226a2f/B in TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:28,452 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. files: [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/d0b8fc44900d438dafe9b526ef2971f3, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/24dd8082057a4581a1fbd40574a574f2, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/7d7554db47c0437aafc12ef7aa6970ad] 2024-11-14T09:14:28,452 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/6210f261404d457bba5b21ad46ac3aa9, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/0bcd4c2f6f6f45578fc7eb40b25de789, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/0f7f48ff16c64443af965a9110966736] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp, totalSize=36.0 K 2024-11-14T09:14:28,452 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 6210f261404d457bba5b21ad46ac3aa9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731575664472 2024-11-14T09:14:28,452 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0b8fc44900d438dafe9b526ef2971f3, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731575664472 2024-11-14T09:14:28,453 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bcd4c2f6f6f45578fc7eb40b25de789, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731575665224 2024-11-14T09:14:28,453 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24dd8082057a4581a1fbd40574a574f2, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731575665224 2024-11-14T09:14:28,453 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f7f48ff16c64443af965a9110966736, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731575665895 2024-11-14T09:14:28,454 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d7554db47c0437aafc12ef7aa6970ad, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731575665895 2024-11-14T09:14:28,468 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a7663ca41075544279db903393226a2f] 2024-11-14T09:14:28,469 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7663ca41075544279db903393226a2f#B#compaction#159 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:14:28,469 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/0cbb2b5e91a641049d3fc308fda76dab is 50, key is test_row_0/B:col10/1731575665895/Put/seqid=0 2024-11-14T09:14:28,471 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411149af948764c8948fca2f43d8074648e13_a7663ca41075544279db903393226a2f store=[table=TestAcidGuarantees family=A region=a7663ca41075544279db903393226a2f] 2024-11-14T09:14:28,473 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411149af948764c8948fca2f43d8074648e13_a7663ca41075544279db903393226a2f, store=[table=TestAcidGuarantees family=A region=a7663ca41075544279db903393226a2f] 2024-11-14T09:14:28,473 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411149af948764c8948fca2f43d8074648e13_a7663ca41075544279db903393226a2f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a7663ca41075544279db903393226a2f] 2024-11-14T09:14:28,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742014_1190 (size=12663) 2024-11-14T09:14:28,496 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-14T09:14:28,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:28,497 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing a7663ca41075544279db903393226a2f 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-14T09:14:28,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=A 2024-11-14T09:14:28,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:28,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=B 2024-11-14T09:14:28,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:28,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=C 2024-11-14T09:14:28,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:28,508 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/0cbb2b5e91a641049d3fc308fda76dab as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/0cbb2b5e91a641049d3fc308fda76dab 2024-11-14T09:14:28,519 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7663ca41075544279db903393226a2f/B of a7663ca41075544279db903393226a2f into 0cbb2b5e91a641049d3fc308fda76dab(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:14:28,519 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7663ca41075544279db903393226a2f: 2024-11-14T09:14:28,519 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., storeName=a7663ca41075544279db903393226a2f/B, priority=13, startTime=1731575668450; duration=0sec 2024-11-14T09:14:28,519 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:14:28,519 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7663ca41075544279db903393226a2f:B 2024-11-14T09:14:28,519 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:14:28,522 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:14:28,522 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): a7663ca41075544279db903393226a2f/C is initiating minor compaction (all files) 2024-11-14T09:14:28,522 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7663ca41075544279db903393226a2f/C in TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:28,522 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/b08275aba6c34e949583c53876920b5f, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/a289daf5fdd746e4a4026d539b81b8bf, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/452cfb5cbfa24b23ba032263a1d7c630] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp, totalSize=36.0 K 2024-11-14T09:14:28,523 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting b08275aba6c34e949583c53876920b5f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731575664472 2024-11-14T09:14:28,523 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting a289daf5fdd746e4a4026d539b81b8bf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731575665224 2024-11-14T09:14:28,523 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 452cfb5cbfa24b23ba032263a1d7c630, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731575665895 2024-11-14T09:14:28,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114183429e7b7c7478c8452d58ddcfd238a_a7663ca41075544279db903393226a2f is 50, key is test_row_0/A:col10/1731575667076/Put/seqid=0 2024-11-14T09:14:28,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742015_1191 (size=4469) 2024-11-14T09:14:28,535 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7663ca41075544279db903393226a2f#C#compaction#162 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:14:28,535 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/03a90dec1814426aa0c3f54efcba964f is 50, key is test_row_0/C:col10/1731575665895/Put/seqid=0 2024-11-14T09:14:28,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742017_1193 (size=12663) 2024-11-14T09:14:28,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742016_1192 (size=12304) 2024-11-14T09:14:28,571 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/03a90dec1814426aa0c3f54efcba964f as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/03a90dec1814426aa0c3f54efcba964f 2024-11-14T09:14:28,578 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7663ca41075544279db903393226a2f/C of a7663ca41075544279db903393226a2f into 03a90dec1814426aa0c3f54efcba964f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:14:28,578 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7663ca41075544279db903393226a2f: 2024-11-14T09:14:28,578 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., storeName=a7663ca41075544279db903393226a2f/C, priority=13, startTime=1731575668450; duration=0sec 2024-11-14T09:14:28,579 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:28,579 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7663ca41075544279db903393226a2f:C 2024-11-14T09:14:28,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-14T09:14:28,929 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7663ca41075544279db903393226a2f#A#compaction#160 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:14:28,930 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/83bafa3e4c1d496c96ccf339b58d53eb is 175, key is test_row_0/A:col10/1731575665895/Put/seqid=0 2024-11-14T09:14:28,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:28,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-14T09:14:28,970 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114183429e7b7c7478c8452d58ddcfd238a_a7663ca41075544279db903393226a2f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114183429e7b7c7478c8452d58ddcfd238a_a7663ca41075544279db903393226a2f 2024-11-14T09:14:28,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/cd72e5a6f27e4933a562ad745f34d19c, store: [table=TestAcidGuarantees family=A region=a7663ca41075544279db903393226a2f] 2024-11-14T09:14:28,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/cd72e5a6f27e4933a562ad745f34d19c is 175, key is test_row_0/A:col10/1731575667076/Put/seqid=0 2024-11-14T09:14:28,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742018_1194 (size=31617) 2024-11-14T09:14:28,990 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/83bafa3e4c1d496c96ccf339b58d53eb as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/83bafa3e4c1d496c96ccf339b58d53eb 2024-11-14T09:14:28,995 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7663ca41075544279db903393226a2f/A of a7663ca41075544279db903393226a2f into 83bafa3e4c1d496c96ccf339b58d53eb(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:14:28,995 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7663ca41075544279db903393226a2f: 2024-11-14T09:14:28,995 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., storeName=a7663ca41075544279db903393226a2f/A, priority=13, startTime=1731575668450; duration=0sec 2024-11-14T09:14:28,995 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:28,995 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7663ca41075544279db903393226a2f:A 2024-11-14T09:14:29,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742019_1195 (size=31105) 2024-11-14T09:14:29,012 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=239, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/cd72e5a6f27e4933a562ad745f34d19c 2024-11-14T09:14:29,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/aa0cb24bc9134da08aae63d70dba4903 is 50, key is test_row_0/B:col10/1731575667076/Put/seqid=0 2024-11-14T09:14:29,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742020_1196 (size=12151) 2024-11-14T09:14:29,063 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/aa0cb24bc9134da08aae63d70dba4903 2024-11-14T09:14:29,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/7285a6a799374b4f81c0fe4cbda1b937 is 50, key is test_row_0/C:col10/1731575667076/Put/seqid=0 2024-11-14T09:14:29,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742021_1197 (size=12151) 2024-11-14T09:14:29,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. as already flushing 2024-11-14T09:14:29,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on a7663ca41075544279db903393226a2f 2024-11-14T09:14:29,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:29,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1731575729283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:29,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:29,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:29,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:29,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:29,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50700 deadline: 1731575729287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:29,292 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:29,292 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:29,292 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:29,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:29,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1731575729396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:29,399 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:29,399 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:29,399 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:29,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:29,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50700 deadline: 1731575729400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:29,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:29,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:29,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:29,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-14T09:14:29,573 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/7285a6a799374b4f81c0fe4cbda1b937 2024-11-14T09:14:29,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/cd72e5a6f27e4933a562ad745f34d19c as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/cd72e5a6f27e4933a562ad745f34d19c 2024-11-14T09:14:29,590 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/cd72e5a6f27e4933a562ad745f34d19c, entries=150, sequenceid=239, filesize=30.4 K 2024-11-14T09:14:29,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/aa0cb24bc9134da08aae63d70dba4903 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/aa0cb24bc9134da08aae63d70dba4903 2024-11-14T09:14:29,597 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/aa0cb24bc9134da08aae63d70dba4903, entries=150, sequenceid=239, filesize=11.9 K 2024-11-14T09:14:29,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/7285a6a799374b4f81c0fe4cbda1b937 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/7285a6a799374b4f81c0fe4cbda1b937 2024-11-14T09:14:29,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:29,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1731575729606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:29,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:29,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:29,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:29,614 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/7285a6a799374b4f81c0fe4cbda1b937, entries=150, sequenceid=239, filesize=11.9 K 2024-11-14T09:14:29,615 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for a7663ca41075544279db903393226a2f in 1118ms, sequenceid=239, compaction requested=false 2024-11-14T09:14:29,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for a7663ca41075544279db903393226a2f: 2024-11-14T09:14:29,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:29,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-14T09:14:29,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-14T09:14:29,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on a7663ca41075544279db903393226a2f 2024-11-14T09:14:29,618 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a7663ca41075544279db903393226a2f 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-14T09:14:29,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=A 2024-11-14T09:14:29,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:29,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=B 2024-11-14T09:14:29,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:29,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=C 2024-11-14T09:14:29,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:29,619 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-14T09:14:29,619 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2760 sec 2024-11-14T09:14:29,622 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.2840 sec 2024-11-14T09:14:29,654 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e44c1e6ea4fc438babc20fc0ad0c6723_a7663ca41075544279db903393226a2f is 50, key is test_row_0/A:col10/1731575669282/Put/seqid=0 2024-11-14T09:14:29,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742022_1198 (size=14794) 2024-11-14T09:14:29,660 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:29,667 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e44c1e6ea4fc438babc20fc0ad0c6723_a7663ca41075544279db903393226a2f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e44c1e6ea4fc438babc20fc0ad0c6723_a7663ca41075544279db903393226a2f 2024-11-14T09:14:29,668 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/efa223b6872643b4a2284d039fccff01, store: [table=TestAcidGuarantees family=A region=a7663ca41075544279db903393226a2f] 2024-11-14T09:14:29,669 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/efa223b6872643b4a2284d039fccff01 is 175, key is test_row_0/A:col10/1731575669282/Put/seqid=0 2024-11-14T09:14:29,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742023_1199 (size=39749) 2024-11-14T09:14:29,684 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=256, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/efa223b6872643b4a2284d039fccff01 2024-11-14T09:14:29,694 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/1fefa921385b4e2a97daea82215d34a7 is 50, key is test_row_0/B:col10/1731575669282/Put/seqid=0 2024-11-14T09:14:29,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:29,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50700 deadline: 1731575729709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:29,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:29,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:29,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:29,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742024_1200 (size=12151) 2024-11-14T09:14:29,741 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/1fefa921385b4e2a97daea82215d34a7 2024-11-14T09:14:29,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/edd99b2ad3644db6965909704a176ccc is 50, key is test_row_0/C:col10/1731575669282/Put/seqid=0 2024-11-14T09:14:29,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742025_1201 (size=12151) 2024-11-14T09:14:29,771 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/edd99b2ad3644db6965909704a176ccc 2024-11-14T09:14:29,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/efa223b6872643b4a2284d039fccff01 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/efa223b6872643b4a2284d039fccff01 2024-11-14T09:14:29,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/efa223b6872643b4a2284d039fccff01, entries=200, sequenceid=256, filesize=38.8 K 2024-11-14T09:14:29,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/1fefa921385b4e2a97daea82215d34a7 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/1fefa921385b4e2a97daea82215d34a7 2024-11-14T09:14:29,793 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/1fefa921385b4e2a97daea82215d34a7, entries=150, sequenceid=256, filesize=11.9 K 2024-11-14T09:14:29,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/edd99b2ad3644db6965909704a176ccc as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/edd99b2ad3644db6965909704a176ccc 2024-11-14T09:14:29,799 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/edd99b2ad3644db6965909704a176ccc, entries=150, sequenceid=256, filesize=11.9 K 2024-11-14T09:14:29,800 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for a7663ca41075544279db903393226a2f in 182ms, sequenceid=256, compaction requested=true 2024-11-14T09:14:29,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a7663ca41075544279db903393226a2f: 2024-11-14T09:14:29,800 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:14:29,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7663ca41075544279db903393226a2f:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:14:29,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:29,801 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:14:29,801 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102471 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:14:29,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7663ca41075544279db903393226a2f:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:14:29,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:29,801 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): a7663ca41075544279db903393226a2f/A is initiating minor compaction (all files) 2024-11-14T09:14:29,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7663ca41075544279db903393226a2f:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:14:29,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:14:29,801 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7663ca41075544279db903393226a2f/A in TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:29,802 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/83bafa3e4c1d496c96ccf339b58d53eb, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/cd72e5a6f27e4933a562ad745f34d19c, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/efa223b6872643b4a2284d039fccff01] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp, totalSize=100.1 K 2024-11-14T09:14:29,802 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:29,802 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. files: [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/83bafa3e4c1d496c96ccf339b58d53eb, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/cd72e5a6f27e4933a562ad745f34d19c, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/efa223b6872643b4a2284d039fccff01] 2024-11-14T09:14:29,802 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83bafa3e4c1d496c96ccf339b58d53eb, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731575665895 2024-11-14T09:14:29,803 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:14:29,803 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd72e5a6f27e4933a562ad745f34d19c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731575667076 2024-11-14T09:14:29,803 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): a7663ca41075544279db903393226a2f/B is initiating minor compaction (all files) 2024-11-14T09:14:29,803 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7663ca41075544279db903393226a2f/B in TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:29,803 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/0cbb2b5e91a641049d3fc308fda76dab, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/aa0cb24bc9134da08aae63d70dba4903, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/1fefa921385b4e2a97daea82215d34a7] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp, totalSize=36.1 K 2024-11-14T09:14:29,803 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting efa223b6872643b4a2284d039fccff01, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731575669279 2024-11-14T09:14:29,803 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cbb2b5e91a641049d3fc308fda76dab, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731575665895 2024-11-14T09:14:29,805 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting aa0cb24bc9134da08aae63d70dba4903, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731575667076 2024-11-14T09:14:29,805 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fefa921385b4e2a97daea82215d34a7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731575669282 2024-11-14T09:14:29,821 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a7663ca41075544279db903393226a2f] 2024-11-14T09:14:29,824 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7663ca41075544279db903393226a2f#B#compaction#169 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:14:29,824 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/9ca6252dcc384ad8a4a64221e2b1eda7 is 50, key is test_row_0/B:col10/1731575669282/Put/seqid=0 2024-11-14T09:14:29,832 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411144ce7d1dcb00a4647a4c1b9b4d10db50f_a7663ca41075544279db903393226a2f store=[table=TestAcidGuarantees family=A region=a7663ca41075544279db903393226a2f] 2024-11-14T09:14:29,834 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411144ce7d1dcb00a4647a4c1b9b4d10db50f_a7663ca41075544279db903393226a2f, store=[table=TestAcidGuarantees family=A region=a7663ca41075544279db903393226a2f] 2024-11-14T09:14:29,834 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144ce7d1dcb00a4647a4c1b9b4d10db50f_a7663ca41075544279db903393226a2f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a7663ca41075544279db903393226a2f] 2024-11-14T09:14:29,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on a7663ca41075544279db903393226a2f 2024-11-14T09:14:29,844 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a7663ca41075544279db903393226a2f 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-14T09:14:29,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=A 2024-11-14T09:14:29,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:29,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=B 2024-11-14T09:14:29,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:29,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=C 2024-11-14T09:14:29,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:29,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742026_1202 (size=12765) 2024-11-14T09:14:29,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742027_1203 (size=4469) 2024-11-14T09:14:29,887 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7663ca41075544279db903393226a2f#A#compaction#168 average throughput is 0.38 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:14:29,888 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/bba77d72baf847f4ba9f582c0e8f1daf is 175, key is test_row_0/A:col10/1731575669282/Put/seqid=0 2024-11-14T09:14:29,895 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/9ca6252dcc384ad8a4a64221e2b1eda7 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/9ca6252dcc384ad8a4a64221e2b1eda7 2024-11-14T09:14:29,903 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7663ca41075544279db903393226a2f/B of a7663ca41075544279db903393226a2f into 9ca6252dcc384ad8a4a64221e2b1eda7(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:14:29,903 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7663ca41075544279db903393226a2f: 2024-11-14T09:14:29,903 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., storeName=a7663ca41075544279db903393226a2f/B, priority=13, startTime=1731575669801; duration=0sec 2024-11-14T09:14:29,903 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:14:29,903 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7663ca41075544279db903393226a2f:B 2024-11-14T09:14:29,904 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:14:29,908 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:14:29,908 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): a7663ca41075544279db903393226a2f/C is initiating minor compaction (all files) 2024-11-14T09:14:29,908 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7663ca41075544279db903393226a2f/C in TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:29,908 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/03a90dec1814426aa0c3f54efcba964f, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/7285a6a799374b4f81c0fe4cbda1b937, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/edd99b2ad3644db6965909704a176ccc] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp, totalSize=36.1 K 2024-11-14T09:14:29,908 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 03a90dec1814426aa0c3f54efcba964f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731575665895 2024-11-14T09:14:29,909 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 7285a6a799374b4f81c0fe4cbda1b937, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731575667076 2024-11-14T09:14:29,909 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411142fe4a1e05cc04505b81cf18331e3b060_a7663ca41075544279db903393226a2f is 50, key is test_row_0/A:col10/1731575669831/Put/seqid=0 2024-11-14T09:14:29,918 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting edd99b2ad3644db6965909704a176ccc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731575669282 2024-11-14T09:14:29,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742028_1204 (size=31719) 2024-11-14T09:14:29,928 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/bba77d72baf847f4ba9f582c0e8f1daf as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/bba77d72baf847f4ba9f582c0e8f1daf 2024-11-14T09:14:29,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:29,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50700 deadline: 1731575729930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:29,936 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:29,936 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:29,936 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:29,939 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7663ca41075544279db903393226a2f/A of a7663ca41075544279db903393226a2f into bba77d72baf847f4ba9f582c0e8f1daf(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:14:29,939 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7663ca41075544279db903393226a2f: 2024-11-14T09:14:29,939 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., storeName=a7663ca41075544279db903393226a2f/A, priority=13, startTime=1731575669800; duration=0sec 2024-11-14T09:14:29,939 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:29,939 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7663ca41075544279db903393226a2f:A 2024-11-14T09:14:29,941 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7663ca41075544279db903393226a2f#C#compaction#171 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:14:29,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:29,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1731575729934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:29,942 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/c501133f7ea04b4ba8696b6e05ab7e1a is 50, key is test_row_0/C:col10/1731575669282/Put/seqid=0 2024-11-14T09:14:29,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:29,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:29,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:29,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742029_1205 (size=14994) 2024-11-14T09:14:29,958 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:29,963 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411142fe4a1e05cc04505b81cf18331e3b060_a7663ca41075544279db903393226a2f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411142fe4a1e05cc04505b81cf18331e3b060_a7663ca41075544279db903393226a2f 2024-11-14T09:14:29,964 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/6dffcaedb00d4a0584089d7d3d4b85d0, store: [table=TestAcidGuarantees family=A region=a7663ca41075544279db903393226a2f] 2024-11-14T09:14:29,965 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/6dffcaedb00d4a0584089d7d3d4b85d0 is 175, key is test_row_0/A:col10/1731575669831/Put/seqid=0 2024-11-14T09:14:29,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742030_1206 (size=12765) 2024-11-14T09:14:29,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742031_1207 (size=39949) 2024-11-14T09:14:29,999 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=275, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/6dffcaedb00d4a0584089d7d3d4b85d0 2024-11-14T09:14:29,999 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/c501133f7ea04b4ba8696b6e05ab7e1a as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/c501133f7ea04b4ba8696b6e05ab7e1a 2024-11-14T09:14:30,014 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7663ca41075544279db903393226a2f/C of a7663ca41075544279db903393226a2f into c501133f7ea04b4ba8696b6e05ab7e1a(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:14:30,014 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7663ca41075544279db903393226a2f: 2024-11-14T09:14:30,014 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., storeName=a7663ca41075544279db903393226a2f/C, priority=13, startTime=1731575669801; duration=0sec 2024-11-14T09:14:30,014 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:30,014 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7663ca41075544279db903393226a2f:C 2024-11-14T09:14:30,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/d93b9a3797884d5aaae4081143624951 is 50, key is test_row_0/B:col10/1731575669831/Put/seqid=0 2024-11-14T09:14:30,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742032_1208 (size=12301) 2024-11-14T09:14:30,042 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/d93b9a3797884d5aaae4081143624951 2024-11-14T09:14:30,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:30,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50700 deadline: 1731575730045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:30,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:30,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1731575730045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:30,047 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:30,047 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:30,047 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:30,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:30,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:30,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:30,052 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/00a0421d3ec1461da0ba008cb657f372 is 50, key is test_row_0/C:col10/1731575669831/Put/seqid=0 2024-11-14T09:14:30,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742033_1209 (size=12301) 2024-11-14T09:14:30,067 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/00a0421d3ec1461da0ba008cb657f372 2024-11-14T09:14:30,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/6dffcaedb00d4a0584089d7d3d4b85d0 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/6dffcaedb00d4a0584089d7d3d4b85d0 2024-11-14T09:14:30,087 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/6dffcaedb00d4a0584089d7d3d4b85d0, entries=200, sequenceid=275, filesize=39.0 K 2024-11-14T09:14:30,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/d93b9a3797884d5aaae4081143624951 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/d93b9a3797884d5aaae4081143624951 2024-11-14T09:14:30,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,094 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/d93b9a3797884d5aaae4081143624951, entries=150, sequenceid=275, filesize=12.0 K 2024-11-14T09:14:30,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/00a0421d3ec1461da0ba008cb657f372 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/00a0421d3ec1461da0ba008cb657f372 2024-11-14T09:14:30,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,108 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/00a0421d3ec1461da0ba008cb657f372, entries=150, sequenceid=275, filesize=12.0 K 2024-11-14T09:14:30,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,110 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for a7663ca41075544279db903393226a2f in 266ms, sequenceid=275, compaction requested=false 2024-11-14T09:14:30,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a7663ca41075544279db903393226a2f: 2024-11-14T09:14:30,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,265 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a7663ca41075544279db903393226a2f 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-14T09:14:30,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=A 2024-11-14T09:14:30,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:30,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=B 2024-11-14T09:14:30,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:30,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=C 2024-11-14T09:14:30,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:30,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on a7663ca41075544279db903393226a2f 2024-11-14T09:14:30,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111445ad60d1ebbe4b0dac386b2c6fc51d44_a7663ca41075544279db903393226a2f is 50, key is test_row_0/A:col10/1731575670263/Put/seqid=0 2024-11-14T09:14:30,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742034_1210 (size=12454) 2024-11-14T09:14:30,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,329 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,335 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111445ad60d1ebbe4b0dac386b2c6fc51d44_a7663ca41075544279db903393226a2f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111445ad60d1ebbe4b0dac386b2c6fc51d44_a7663ca41075544279db903393226a2f 2024-11-14T09:14:30,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,336 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/31315ee9288141158fc7fa1531585b0c, store: [table=TestAcidGuarantees family=A region=a7663ca41075544279db903393226a2f] 2024-11-14T09:14:30,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,337 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/31315ee9288141158fc7fa1531585b0c is 175, key is test_row_0/A:col10/1731575670263/Put/seqid=0 2024-11-14T09:14:30,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:30,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:30,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1731575730363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:30,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:30,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:30,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:30,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:30,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50700 deadline: 1731575730371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:30,378 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:30,378 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:30,378 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:30,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742035_1211 (size=31251) 2024-11-14T09:14:30,380 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=292, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/31315ee9288141158fc7fa1531585b0c 2024-11-14T09:14:30,393 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/5999581b11ce45a1ab5264316536b9cd is 50, key is test_row_0/B:col10/1731575670263/Put/seqid=0 2024-11-14T09:14:30,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742036_1212 (size=9857) 2024-11-14T09:14:30,407 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/5999581b11ce45a1ab5264316536b9cd 2024-11-14T09:14:30,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/e6a1e983d07a4eba9042cf855fa705e8 is 50, key is test_row_0/C:col10/1731575670263/Put/seqid=0 2024-11-14T09:14:30,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742037_1213 (size=9857) 2024-11-14T09:14:30,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:30,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1731575730474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:30,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:30,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:30,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:30,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-14T09:14:30,485 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:14:30,486 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:14:30,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-14T09:14:30,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:30,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50700 deadline: 1731575730485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:30,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:30,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:30,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:30,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-14T09:14:30,488 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:14:30,489 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:14:30,489 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:14:30,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-14T09:14:30,643 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-14T09:14:30,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:30,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. as already flushing 2024-11-14T09:14:30,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:30,643 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:30,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:30,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:30,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:30,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1731575730685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:30,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:30,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:30,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:30,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:30,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50700 deadline: 1731575730695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:30,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:30,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:30,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:30,805 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-14T09:14:30,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-14T09:14:30,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:30,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. as already flushing 2024-11-14T09:14:30,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:30,805 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:30,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:30,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:30,872 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/e6a1e983d07a4eba9042cf855fa705e8 2024-11-14T09:14:30,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/31315ee9288141158fc7fa1531585b0c as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/31315ee9288141158fc7fa1531585b0c 2024-11-14T09:14:30,891 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/31315ee9288141158fc7fa1531585b0c, entries=150, sequenceid=292, filesize=30.5 K 2024-11-14T09:14:30,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/5999581b11ce45a1ab5264316536b9cd as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/5999581b11ce45a1ab5264316536b9cd 2024-11-14T09:14:30,900 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/5999581b11ce45a1ab5264316536b9cd, entries=100, sequenceid=292, filesize=9.6 K 2024-11-14T09:14:30,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/e6a1e983d07a4eba9042cf855fa705e8 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/e6a1e983d07a4eba9042cf855fa705e8 2024-11-14T09:14:30,909 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/e6a1e983d07a4eba9042cf855fa705e8, entries=100, sequenceid=292, filesize=9.6 K 2024-11-14T09:14:30,912 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for a7663ca41075544279db903393226a2f in 646ms, sequenceid=292, compaction requested=true 2024-11-14T09:14:30,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a7663ca41075544279db903393226a2f: 2024-11-14T09:14:30,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7663ca41075544279db903393226a2f:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:14:30,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:30,912 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:14:30,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7663ca41075544279db903393226a2f:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:14:30,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:14:30,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7663ca41075544279db903393226a2f:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:14:30,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T09:14:30,912 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:14:30,914 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34923 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:14:30,914 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102919 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:14:30,914 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): a7663ca41075544279db903393226a2f/B is initiating minor compaction (all files) 2024-11-14T09:14:30,914 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): a7663ca41075544279db903393226a2f/A is initiating minor compaction (all files) 2024-11-14T09:14:30,914 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7663ca41075544279db903393226a2f/B in TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:30,914 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7663ca41075544279db903393226a2f/A in TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:30,914 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/9ca6252dcc384ad8a4a64221e2b1eda7, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/d93b9a3797884d5aaae4081143624951, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/5999581b11ce45a1ab5264316536b9cd] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp, totalSize=34.1 K 2024-11-14T09:14:30,914 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/bba77d72baf847f4ba9f582c0e8f1daf, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/6dffcaedb00d4a0584089d7d3d4b85d0, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/31315ee9288141158fc7fa1531585b0c] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp, totalSize=100.5 K 2024-11-14T09:14:30,914 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:30,914 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. files: [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/bba77d72baf847f4ba9f582c0e8f1daf, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/6dffcaedb00d4a0584089d7d3d4b85d0, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/31315ee9288141158fc7fa1531585b0c] 2024-11-14T09:14:30,915 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ca6252dcc384ad8a4a64221e2b1eda7, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731575669282 2024-11-14T09:14:30,915 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting bba77d72baf847f4ba9f582c0e8f1daf, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731575669282 2024-11-14T09:14:30,916 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting d93b9a3797884d5aaae4081143624951, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731575669692 2024-11-14T09:14:30,916 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 5999581b11ce45a1ab5264316536b9cd, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731575669927 2024-11-14T09:14:30,916 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6dffcaedb00d4a0584089d7d3d4b85d0, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731575669634 2024-11-14T09:14:30,917 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31315ee9288141158fc7fa1531585b0c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731575669927 2024-11-14T09:14:30,938 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a7663ca41075544279db903393226a2f] 2024-11-14T09:14:30,938 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7663ca41075544279db903393226a2f#B#compaction#177 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:14:30,939 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/779a5cdd1c53431eb3e83fe2b0689adc is 50, key is test_row_0/B:col10/1731575670263/Put/seqid=0 2024-11-14T09:14:30,947 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111436a568730fb745e7a6e9ca83b0900ad5_a7663ca41075544279db903393226a2f store=[table=TestAcidGuarantees family=A region=a7663ca41075544279db903393226a2f] 2024-11-14T09:14:30,950 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111436a568730fb745e7a6e9ca83b0900ad5_a7663ca41075544279db903393226a2f, store=[table=TestAcidGuarantees family=A region=a7663ca41075544279db903393226a2f] 2024-11-14T09:14:30,950 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111436a568730fb745e7a6e9ca83b0900ad5_a7663ca41075544279db903393226a2f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a7663ca41075544279db903393226a2f] 2024-11-14T09:14:30,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742039_1215 (size=4469) 2024-11-14T09:14:30,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-14T09:14:30,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:30,961 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing a7663ca41075544279db903393226a2f 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-14T09:14:30,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=A 2024-11-14T09:14:30,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:30,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=B 2024-11-14T09:14:30,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:30,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=C 2024-11-14T09:14:30,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:30,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742038_1214 (size=13017) 2024-11-14T09:14:30,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114f610524d027a4ee5a642e6ea8acbaef6_a7663ca41075544279db903393226a2f is 50, key is test_row_0/A:col10/1731575670361/Put/seqid=0 2024-11-14T09:14:30,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on a7663ca41075544279db903393226a2f 2024-11-14T09:14:30,999 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. as already flushing 2024-11-14T09:14:31,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742040_1216 (size=12454) 2024-11-14T09:14:31,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:31,035 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114f610524d027a4ee5a642e6ea8acbaef6_a7663ca41075544279db903393226a2f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f610524d027a4ee5a642e6ea8acbaef6_a7663ca41075544279db903393226a2f 2024-11-14T09:14:31,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/1071fa53645c499798c93eb8f80faf42, store: [table=TestAcidGuarantees family=A region=a7663ca41075544279db903393226a2f] 2024-11-14T09:14:31,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/1071fa53645c499798c93eb8f80faf42 is 175, key is test_row_0/A:col10/1731575670361/Put/seqid=0 2024-11-14T09:14:31,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:31,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1731575731046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:31,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:31,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:31,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:31,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:31,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50700 deadline: 1731575731047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:31,054 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:31,054 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:31,054 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:31,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742041_1217 (size=31255) 2024-11-14T09:14:31,056 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=310, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/1071fa53645c499798c93eb8f80faf42 2024-11-14T09:14:31,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/bb5f6df933ef47409a4847018afea7b1 is 50, key is test_row_0/B:col10/1731575670361/Put/seqid=0 2024-11-14T09:14:31,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742042_1218 (size=12301) 2024-11-14T09:14:31,071 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/bb5f6df933ef47409a4847018afea7b1 2024-11-14T09:14:31,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/0dc1cea199f2406ebd9f730fb563824f is 50, key is test_row_0/C:col10/1731575670361/Put/seqid=0 2024-11-14T09:14:31,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742043_1219 (size=12301) 2024-11-14T09:14:31,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-14T09:14:31,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:31,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1731575731155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:31,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:31,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:31,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:31,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:31,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50700 deadline: 1731575731165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:31,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:31,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:31,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:31,361 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7663ca41075544279db903393226a2f#A#compaction#178 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:14:31,362 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/48c982c6c21e42e892486985275fb7be is 175, key is test_row_0/A:col10/1731575670263/Put/seqid=0 2024-11-14T09:14:31,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:31,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1731575731365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:31,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:31,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:31,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:31,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742044_1220 (size=32078) 2024-11-14T09:14:31,376 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/779a5cdd1c53431eb3e83fe2b0689adc as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/779a5cdd1c53431eb3e83fe2b0689adc 2024-11-14T09:14:31,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:31,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50700 deadline: 1731575731375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:31,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:31,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:31,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:31,384 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7663ca41075544279db903393226a2f/B of a7663ca41075544279db903393226a2f into 779a5cdd1c53431eb3e83fe2b0689adc(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:14:31,384 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7663ca41075544279db903393226a2f: 2024-11-14T09:14:31,384 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., storeName=a7663ca41075544279db903393226a2f/B, priority=13, startTime=1731575670912; duration=0sec 2024-11-14T09:14:31,384 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:14:31,385 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7663ca41075544279db903393226a2f:B 2024-11-14T09:14:31,385 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:14:31,386 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34923 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:14:31,386 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): a7663ca41075544279db903393226a2f/C is initiating minor compaction (all files) 2024-11-14T09:14:31,386 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7663ca41075544279db903393226a2f/C in TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:31,386 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/c501133f7ea04b4ba8696b6e05ab7e1a, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/00a0421d3ec1461da0ba008cb657f372, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/e6a1e983d07a4eba9042cf855fa705e8] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp, totalSize=34.1 K 2024-11-14T09:14:31,386 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting c501133f7ea04b4ba8696b6e05ab7e1a, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731575669282 2024-11-14T09:14:31,387 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 00a0421d3ec1461da0ba008cb657f372, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731575669692 2024-11-14T09:14:31,388 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting e6a1e983d07a4eba9042cf855fa705e8, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731575669927 2024-11-14T09:14:31,403 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7663ca41075544279db903393226a2f#C#compaction#182 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:14:31,403 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/3d4d83b7acb34219b7fb6a977580ef9a is 50, key is test_row_0/C:col10/1731575670263/Put/seqid=0 2024-11-14T09:14:31,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742045_1221 (size=13017) 2024-11-14T09:14:31,418 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/3d4d83b7acb34219b7fb6a977580ef9a as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/3d4d83b7acb34219b7fb6a977580ef9a 2024-11-14T09:14:31,425 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7663ca41075544279db903393226a2f/C of a7663ca41075544279db903393226a2f into 3d4d83b7acb34219b7fb6a977580ef9a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:14:31,425 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7663ca41075544279db903393226a2f: 2024-11-14T09:14:31,425 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., storeName=a7663ca41075544279db903393226a2f/C, priority=13, startTime=1731575670912; duration=0sec 2024-11-14T09:14:31,426 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:31,426 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7663ca41075544279db903393226a2f:C 2024-11-14T09:14:31,487 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/0dc1cea199f2406ebd9f730fb563824f 2024-11-14T09:14:31,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/1071fa53645c499798c93eb8f80faf42 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/1071fa53645c499798c93eb8f80faf42 2024-11-14T09:14:31,497 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/1071fa53645c499798c93eb8f80faf42, entries=150, sequenceid=310, filesize=30.5 K 2024-11-14T09:14:31,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/bb5f6df933ef47409a4847018afea7b1 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/bb5f6df933ef47409a4847018afea7b1 2024-11-14T09:14:31,502 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/bb5f6df933ef47409a4847018afea7b1, entries=150, sequenceid=310, filesize=12.0 K 2024-11-14T09:14:31,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/0dc1cea199f2406ebd9f730fb563824f as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/0dc1cea199f2406ebd9f730fb563824f 2024-11-14T09:14:31,509 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/0dc1cea199f2406ebd9f730fb563824f, entries=150, sequenceid=310, filesize=12.0 K 2024-11-14T09:14:31,512 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for a7663ca41075544279db903393226a2f in 551ms, sequenceid=310, compaction requested=false 2024-11-14T09:14:31,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for a7663ca41075544279db903393226a2f: 2024-11-14T09:14:31,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:31,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-14T09:14:31,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-14T09:14:31,519 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-14T09:14:31,519 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0250 sec 2024-11-14T09:14:31,521 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.0340 sec 2024-11-14T09:14:31,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-14T09:14:31,625 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:14:31,626 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:14:31,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-14T09:14:31,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-14T09:14:31,629 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:14:31,629 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:14:31,630 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:14:31,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on a7663ca41075544279db903393226a2f 2024-11-14T09:14:31,677 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a7663ca41075544279db903393226a2f 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-14T09:14:31,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=A 2024-11-14T09:14:31,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:31,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=B 2024-11-14T09:14:31,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:31,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=C 2024-11-14T09:14:31,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:31,686 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114094bce857cb243dcb3083d3a86030fca_a7663ca41075544279db903393226a2f is 50, key is test_row_0/A:col10/1731575671008/Put/seqid=0 2024-11-14T09:14:31,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742046_1222 (size=14994) 2024-11-14T09:14:31,691 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:31,700 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114094bce857cb243dcb3083d3a86030fca_a7663ca41075544279db903393226a2f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114094bce857cb243dcb3083d3a86030fca_a7663ca41075544279db903393226a2f 2024-11-14T09:14:31,701 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/74d14dc748644c4aa33b67b02400b0f3, store: [table=TestAcidGuarantees family=A region=a7663ca41075544279db903393226a2f] 2024-11-14T09:14:31,702 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/74d14dc748644c4aa33b67b02400b0f3 is 175, key is test_row_0/A:col10/1731575671008/Put/seqid=0 2024-11-14T09:14:31,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742047_1223 (size=39949) 2024-11-14T09:14:31,707 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=327, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/74d14dc748644c4aa33b67b02400b0f3 2024-11-14T09:14:31,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/7fb6b2a1654b4c90a9a05be6d6b1a0fc is 50, key is test_row_0/B:col10/1731575671008/Put/seqid=0 2024-11-14T09:14:31,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742048_1224 (size=12301) 2024-11-14T09:14:31,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:31,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50700 deadline: 1731575731722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:31,726 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:31,726 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:31,726 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:31,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:31,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1731575731726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:31,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:31,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:31,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:31,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-14T09:14:31,776 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/48c982c6c21e42e892486985275fb7be as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/48c982c6c21e42e892486985275fb7be 2024-11-14T09:14:31,782 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7663ca41075544279db903393226a2f/A of a7663ca41075544279db903393226a2f into 48c982c6c21e42e892486985275fb7be(size=31.3 K), total size for store is 61.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:14:31,782 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7663ca41075544279db903393226a2f: 2024-11-14T09:14:31,782 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., storeName=a7663ca41075544279db903393226a2f/A, priority=13, startTime=1731575670912; duration=0sec 2024-11-14T09:14:31,782 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:31,782 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7663ca41075544279db903393226a2f:A 2024-11-14T09:14:31,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-14T09:14:31,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:31,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. as already flushing 2024-11-14T09:14:31,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:31,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:31,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:31,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:31,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:31,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50700 deadline: 1731575731835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:31,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:31,841 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:31,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1731575731835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:31,841 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:31,841 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:31,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:31,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:31,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:31,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-14T09:14:31,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:31,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. as already flushing 2024-11-14T09:14:31,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:31,936 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:31,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:31,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:31,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-14T09:14:32,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:32,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50700 deadline: 1731575732045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:32,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:32,049 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:32,049 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:32,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1731575732045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:32,049 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:32,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:32,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:32,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:32,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-14T09:14:32,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:32,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. as already flushing 2024-11-14T09:14:32,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:32,090 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:32,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:32,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:32,124 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/7fb6b2a1654b4c90a9a05be6d6b1a0fc 2024-11-14T09:14:32,145 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/0fbc86b3a7c14691a243822406e8ff32 is 50, key is test_row_0/C:col10/1731575671008/Put/seqid=0 2024-11-14T09:14:32,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742049_1225 (size=12301) 2024-11-14T09:14:32,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-14T09:14:32,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:32,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. as already flushing 2024-11-14T09:14:32,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:32,244 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:32,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:32,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:32,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-14T09:14:32,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:32,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50700 deadline: 1731575732354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:32,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:32,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1731575732354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:32,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:32,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:32,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:32,359 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:32,359 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:32,359 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:32,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-14T09:14:32,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:32,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. as already flushing 2024-11-14T09:14:32,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:32,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:32,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:32,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:32,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:32,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50728 deadline: 1731575732415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:32,420 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_0' in a7663ca41075544279db903393226a2f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:14:32,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:32,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:32,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:32,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:32,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731575732434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:32,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:32,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50676 deadline: 1731575732435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:32,438 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_0' in a7663ca41075544279db903393226a2f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8239 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:14:32,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:32,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:32,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:32,439 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_2' in a7663ca41075544279db903393226a2f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8241 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:14:32,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:32,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:32,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:32,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-14T09:14:32,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:32,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. as already flushing 2024-11-14T09:14:32,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:32,551 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:32,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:32,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:32,568 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/0fbc86b3a7c14691a243822406e8ff32 2024-11-14T09:14:32,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/74d14dc748644c4aa33b67b02400b0f3 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/74d14dc748644c4aa33b67b02400b0f3 2024-11-14T09:14:32,579 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/74d14dc748644c4aa33b67b02400b0f3, entries=200, sequenceid=327, filesize=39.0 K 2024-11-14T09:14:32,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/7fb6b2a1654b4c90a9a05be6d6b1a0fc as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/7fb6b2a1654b4c90a9a05be6d6b1a0fc 2024-11-14T09:14:32,587 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/7fb6b2a1654b4c90a9a05be6d6b1a0fc, entries=150, sequenceid=327, filesize=12.0 K 2024-11-14T09:14:32,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/0fbc86b3a7c14691a243822406e8ff32 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/0fbc86b3a7c14691a243822406e8ff32 2024-11-14T09:14:32,594 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/0fbc86b3a7c14691a243822406e8ff32, entries=150, sequenceid=327, filesize=12.0 K 2024-11-14T09:14:32,594 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for a7663ca41075544279db903393226a2f in 917ms, sequenceid=327, compaction requested=true 2024-11-14T09:14:32,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a7663ca41075544279db903393226a2f: 2024-11-14T09:14:32,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7663ca41075544279db903393226a2f:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:14:32,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:32,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7663ca41075544279db903393226a2f:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:14:32,595 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:14:32,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:32,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7663ca41075544279db903393226a2f:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:14:32,595 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:14:32,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:14:32,596 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103282 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:14:32,596 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): a7663ca41075544279db903393226a2f/A is initiating minor compaction (all files) 2024-11-14T09:14:32,596 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:14:32,596 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7663ca41075544279db903393226a2f/A in TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:32,596 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): a7663ca41075544279db903393226a2f/B is initiating minor compaction (all files) 2024-11-14T09:14:32,596 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7663ca41075544279db903393226a2f/B in TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:32,596 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/48c982c6c21e42e892486985275fb7be, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/1071fa53645c499798c93eb8f80faf42, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/74d14dc748644c4aa33b67b02400b0f3] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp, totalSize=100.9 K 2024-11-14T09:14:32,596 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:32,596 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/779a5cdd1c53431eb3e83fe2b0689adc, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/bb5f6df933ef47409a4847018afea7b1, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/7fb6b2a1654b4c90a9a05be6d6b1a0fc] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp, totalSize=36.7 K 2024-11-14T09:14:32,596 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. files: [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/48c982c6c21e42e892486985275fb7be, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/1071fa53645c499798c93eb8f80faf42, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/74d14dc748644c4aa33b67b02400b0f3] 2024-11-14T09:14:32,597 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 779a5cdd1c53431eb3e83fe2b0689adc, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731575669706 2024-11-14T09:14:32,597 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48c982c6c21e42e892486985275fb7be, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731575669706 2024-11-14T09:14:32,597 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1071fa53645c499798c93eb8f80faf42, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731575670331 2024-11-14T09:14:32,597 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting bb5f6df933ef47409a4847018afea7b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731575670331 2024-11-14T09:14:32,598 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74d14dc748644c4aa33b67b02400b0f3, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1731575671008 2024-11-14T09:14:32,598 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fb6b2a1654b4c90a9a05be6d6b1a0fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1731575671008 2024-11-14T09:14:32,607 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7663ca41075544279db903393226a2f#B#compaction#186 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:14:32,607 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/f10d91d68e0c42edb1be0c92c4830d37 is 50, key is test_row_0/B:col10/1731575671008/Put/seqid=0 2024-11-14T09:14:32,609 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a7663ca41075544279db903393226a2f] 2024-11-14T09:14:32,611 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111423008403fd1f4013ad4598561667928e_a7663ca41075544279db903393226a2f store=[table=TestAcidGuarantees family=A region=a7663ca41075544279db903393226a2f] 2024-11-14T09:14:32,613 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111423008403fd1f4013ad4598561667928e_a7663ca41075544279db903393226a2f, store=[table=TestAcidGuarantees family=A region=a7663ca41075544279db903393226a2f] 2024-11-14T09:14:32,614 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111423008403fd1f4013ad4598561667928e_a7663ca41075544279db903393226a2f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a7663ca41075544279db903393226a2f] 2024-11-14T09:14:32,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742050_1226 (size=13119) 2024-11-14T09:14:32,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742051_1227 (size=4469) 2024-11-14T09:14:32,633 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7663ca41075544279db903393226a2f#A#compaction#187 average throughput is 1.02 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:14:32,634 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/29620a757aac42d8b006f5b9515a0456 is 175, key is test_row_0/A:col10/1731575671008/Put/seqid=0 2024-11-14T09:14:32,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742052_1228 (size=32073) 2024-11-14T09:14:32,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-14T09:14:32,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:32,704 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing a7663ca41075544279db903393226a2f 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-14T09:14:32,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=A 2024-11-14T09:14:32,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:32,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=B 2024-11-14T09:14:32,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:32,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=C 2024-11-14T09:14:32,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:32,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111497bdddf95f9f42fd9df617c97fab554d_a7663ca41075544279db903393226a2f is 50, key is test_row_0/A:col10/1731575671716/Put/seqid=0 2024-11-14T09:14:32,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742053_1229 (size=12454) 2024-11-14T09:14:32,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-14T09:14:32,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on a7663ca41075544279db903393226a2f 2024-11-14T09:14:32,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. as already flushing 2024-11-14T09:14:32,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:32,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50700 deadline: 1731575732889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:32,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:32,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:32,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:32,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:32,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1731575732891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:32,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:32,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:32,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:33,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:33,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50700 deadline: 1731575733005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:33,012 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:33,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:33,012 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:33,012 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:33,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1731575733005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:33,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:33,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:33,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:33,025 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/f10d91d68e0c42edb1be0c92c4830d37 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/f10d91d68e0c42edb1be0c92c4830d37 2024-11-14T09:14:33,031 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7663ca41075544279db903393226a2f/B of a7663ca41075544279db903393226a2f into f10d91d68e0c42edb1be0c92c4830d37(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:14:33,031 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7663ca41075544279db903393226a2f: 2024-11-14T09:14:33,031 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., storeName=a7663ca41075544279db903393226a2f/B, priority=13, startTime=1731575672595; duration=0sec 2024-11-14T09:14:33,031 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:14:33,031 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7663ca41075544279db903393226a2f:B 2024-11-14T09:14:33,031 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:14:33,035 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:14:33,035 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): a7663ca41075544279db903393226a2f/C is initiating minor compaction (all files) 2024-11-14T09:14:33,035 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7663ca41075544279db903393226a2f/C in TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:33,035 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/3d4d83b7acb34219b7fb6a977580ef9a, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/0dc1cea199f2406ebd9f730fb563824f, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/0fbc86b3a7c14691a243822406e8ff32] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp, totalSize=36.7 K 2024-11-14T09:14:33,036 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d4d83b7acb34219b7fb6a977580ef9a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731575669706 2024-11-14T09:14:33,036 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 0dc1cea199f2406ebd9f730fb563824f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731575670331 2024-11-14T09:14:33,037 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fbc86b3a7c14691a243822406e8ff32, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1731575671008 2024-11-14T09:14:33,056 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7663ca41075544279db903393226a2f#C#compaction#189 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:14:33,057 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/60799e1f52a549858a75d3b20e0396f2 is 50, key is test_row_0/C:col10/1731575671008/Put/seqid=0 2024-11-14T09:14:33,058 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/29620a757aac42d8b006f5b9515a0456 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/29620a757aac42d8b006f5b9515a0456 2024-11-14T09:14:33,064 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7663ca41075544279db903393226a2f/A of a7663ca41075544279db903393226a2f into 29620a757aac42d8b006f5b9515a0456(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:14:33,064 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7663ca41075544279db903393226a2f: 2024-11-14T09:14:33,064 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., storeName=a7663ca41075544279db903393226a2f/A, priority=13, startTime=1731575672595; duration=0sec 2024-11-14T09:14:33,064 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:33,064 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7663ca41075544279db903393226a2f:A 2024-11-14T09:14:33,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742054_1230 (size=13119) 2024-11-14T09:14:33,088 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/60799e1f52a549858a75d3b20e0396f2 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/60799e1f52a549858a75d3b20e0396f2 2024-11-14T09:14:33,094 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7663ca41075544279db903393226a2f/C of a7663ca41075544279db903393226a2f into 60799e1f52a549858a75d3b20e0396f2(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:14:33,095 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7663ca41075544279db903393226a2f: 2024-11-14T09:14:33,095 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., storeName=a7663ca41075544279db903393226a2f/C, priority=13, startTime=1731575672595; duration=0sec 2024-11-14T09:14:33,095 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:33,095 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7663ca41075544279db903393226a2f:C 2024-11-14T09:14:33,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:33,147 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111497bdddf95f9f42fd9df617c97fab554d_a7663ca41075544279db903393226a2f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111497bdddf95f9f42fd9df617c97fab554d_a7663ca41075544279db903393226a2f 2024-11-14T09:14:33,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/4a26183b8cae46408f9c67fd2c2e214f, store: [table=TestAcidGuarantees family=A region=a7663ca41075544279db903393226a2f] 2024-11-14T09:14:33,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/4a26183b8cae46408f9c67fd2c2e214f is 175, key is test_row_0/A:col10/1731575671716/Put/seqid=0 2024-11-14T09:14:33,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742055_1231 (size=31255) 2024-11-14T09:14:33,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:33,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50700 deadline: 1731575733214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:33,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:33,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:33,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:33,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:33,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1731575733225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:33,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:33,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:33,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:33,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:33,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50700 deadline: 1731575733526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:33,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:33,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:33,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:33,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:33,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1731575733536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:33,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:33,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:33,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:33,556 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=345, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/4a26183b8cae46408f9c67fd2c2e214f 2024-11-14T09:14:33,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/dd1dc19c8ca44ab4b3393fa2e72c05e4 is 50, key is test_row_0/B:col10/1731575671716/Put/seqid=0 2024-11-14T09:14:33,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742056_1232 (size=12301) 2024-11-14T09:14:33,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-14T09:14:33,980 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/dd1dc19c8ca44ab4b3393fa2e72c05e4 2024-11-14T09:14:34,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/3c6b249439ed408ca32ea18ef8399635 is 50, key is test_row_0/C:col10/1731575671716/Put/seqid=0 2024-11-14T09:14:34,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:34,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50700 deadline: 1731575734035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:34,039 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:34,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:34,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:34,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742057_1233 (size=12301) 2024-11-14T09:14:34,047 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/3c6b249439ed408ca32ea18ef8399635 2024-11-14T09:14:34,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:34,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1731575734045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:34,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:34,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:34,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:34,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/4a26183b8cae46408f9c67fd2c2e214f as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/4a26183b8cae46408f9c67fd2c2e214f 2024-11-14T09:14:34,062 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/4a26183b8cae46408f9c67fd2c2e214f, entries=150, sequenceid=345, filesize=30.5 K 2024-11-14T09:14:34,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/dd1dc19c8ca44ab4b3393fa2e72c05e4 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/dd1dc19c8ca44ab4b3393fa2e72c05e4 2024-11-14T09:14:34,081 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/dd1dc19c8ca44ab4b3393fa2e72c05e4, entries=150, sequenceid=345, filesize=12.0 K 2024-11-14T09:14:34,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/3c6b249439ed408ca32ea18ef8399635 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/3c6b249439ed408ca32ea18ef8399635 2024-11-14T09:14:34,087 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/3c6b249439ed408ca32ea18ef8399635, entries=150, sequenceid=345, filesize=12.0 K 2024-11-14T09:14:34,088 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for a7663ca41075544279db903393226a2f in 1384ms, sequenceid=345, compaction requested=false 2024-11-14T09:14:34,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for a7663ca41075544279db903393226a2f: 2024-11-14T09:14:34,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:34,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-14T09:14:34,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-14T09:14:34,093 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-14T09:14:34,094 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4600 sec 2024-11-14T09:14:34,096 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 2.4680 sec 2024-11-14T09:14:35,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on a7663ca41075544279db903393226a2f 2024-11-14T09:14:35,048 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a7663ca41075544279db903393226a2f 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-14T09:14:35,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=A 2024-11-14T09:14:35,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:35,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=B 2024-11-14T09:14:35,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:35,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=C 2024-11-14T09:14:35,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:35,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111484194a96787545728ca4405f3ddf4b70_a7663ca41075544279db903393226a2f is 50, key is test_row_0/A:col10/1731575675046/Put/seqid=0 2024-11-14T09:14:35,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742058_1234 (size=14994) 2024-11-14T09:14:35,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:35,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50700 deadline: 1731575735104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:35,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:35,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:35,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:35,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:35,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1731575735110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:35,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:35,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:35,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:35,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:35,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50700 deadline: 1731575735215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:35,218 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:35,218 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:35,218 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:35,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:35,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1731575735225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:35,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:35,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:35,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:35,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:35,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50700 deadline: 1731575735425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:35,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:35,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:35,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:35,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:35,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1731575735435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:35,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:35,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:35,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:35,475 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:35,481 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111484194a96787545728ca4405f3ddf4b70_a7663ca41075544279db903393226a2f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111484194a96787545728ca4405f3ddf4b70_a7663ca41075544279db903393226a2f 2024-11-14T09:14:35,483 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/ded35735429542b9b060de6ed7a06969, store: [table=TestAcidGuarantees family=A region=a7663ca41075544279db903393226a2f] 2024-11-14T09:14:35,484 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/ded35735429542b9b060de6ed7a06969 is 175, key is test_row_0/A:col10/1731575675046/Put/seqid=0 2024-11-14T09:14:35,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742059_1235 (size=39949) 2024-11-14T09:14:35,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:35,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50700 deadline: 1731575735735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:35,738 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:35,738 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:35,739 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:35,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:35,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1731575735745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:35,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:35,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:35,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:35,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-14T09:14:35,795 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:14:35,796 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:14:35,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-14T09:14:35,799 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:14:35,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-14T09:14:35,801 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:14:35,801 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:14:35,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-14T09:14:35,913 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=364, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/ded35735429542b9b060de6ed7a06969 2024-11-14T09:14:35,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/93a8f9ce588d493b8399d7ad50599a3f is 50, key is test_row_0/B:col10/1731575675046/Put/seqid=0 2024-11-14T09:14:35,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742060_1236 (size=12301) 2024-11-14T09:14:35,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-14T09:14:35,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:35,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. as already flushing 2024-11-14T09:14:35,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:35,955 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:35,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:35,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:36,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-14T09:14:36,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:36,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. as already flushing 2024-11-14T09:14:36,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:36,114 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:36,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:36,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-14T09:14:36,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:36,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:36,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50700 deadline: 1731575736245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:36,247 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:36,247 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:36,247 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:36,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:36,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1731575736254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:36,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:36,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:36,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:14:36,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-14T09:14:36,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:36,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. as already flushing 2024-11-14T09:14:36,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:36,267 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:36,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:36,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:36,346 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/93a8f9ce588d493b8399d7ad50599a3f 2024-11-14T09:14:36,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/206a8c04d5fd4afaa003477bda85293c is 50, key is test_row_0/C:col10/1731575675046/Put/seqid=0 2024-11-14T09:14:36,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742061_1237 (size=12301) 2024-11-14T09:14:36,406 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/206a8c04d5fd4afaa003477bda85293c 2024-11-14T09:14:36,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-14T09:14:36,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:36,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. as already flushing 2024-11-14T09:14:36,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:36,421 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:36,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:36,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:36,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-14T09:14:36,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/ded35735429542b9b060de6ed7a06969 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/ded35735429542b9b060de6ed7a06969 2024-11-14T09:14:36,438 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/ded35735429542b9b060de6ed7a06969, entries=200, sequenceid=364, filesize=39.0 K 2024-11-14T09:14:36,439 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/93a8f9ce588d493b8399d7ad50599a3f as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/93a8f9ce588d493b8399d7ad50599a3f 2024-11-14T09:14:36,444 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/93a8f9ce588d493b8399d7ad50599a3f, entries=150, sequenceid=364, filesize=12.0 K 2024-11-14T09:14:36,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/206a8c04d5fd4afaa003477bda85293c as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/206a8c04d5fd4afaa003477bda85293c 2024-11-14T09:14:36,453 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/206a8c04d5fd4afaa003477bda85293c, entries=150, sequenceid=364, filesize=12.0 K 2024-11-14T09:14:36,454 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for a7663ca41075544279db903393226a2f in 1407ms, sequenceid=364, compaction requested=true 2024-11-14T09:14:36,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a7663ca41075544279db903393226a2f: 2024-11-14T09:14:36,454 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:14:36,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7663ca41075544279db903393226a2f:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:14:36,455 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103277 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:14:36,455 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): a7663ca41075544279db903393226a2f/A is initiating minor compaction (all files) 2024-11-14T09:14:36,455 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7663ca41075544279db903393226a2f/A in TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:36,455 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/29620a757aac42d8b006f5b9515a0456, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/4a26183b8cae46408f9c67fd2c2e214f, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/ded35735429542b9b060de6ed7a06969] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp, totalSize=100.9 K 2024-11-14T09:14:36,456 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:36,456 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. files: [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/29620a757aac42d8b006f5b9515a0456, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/4a26183b8cae46408f9c67fd2c2e214f, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/ded35735429542b9b060de6ed7a06969] 2024-11-14T09:14:36,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:36,456 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:14:36,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7663ca41075544279db903393226a2f:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:14:36,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:36,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7663ca41075544279db903393226a2f:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:14:36,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:14:36,456 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29620a757aac42d8b006f5b9515a0456, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1731575671008 2024-11-14T09:14:36,457 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a26183b8cae46408f9c67fd2c2e214f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731575671716 2024-11-14T09:14:36,458 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting ded35735429542b9b060de6ed7a06969, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731575672883 2024-11-14T09:14:36,459 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:14:36,459 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): a7663ca41075544279db903393226a2f/B is initiating minor compaction (all files) 2024-11-14T09:14:36,459 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7663ca41075544279db903393226a2f/B in TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:36,460 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/f10d91d68e0c42edb1be0c92c4830d37, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/dd1dc19c8ca44ab4b3393fa2e72c05e4, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/93a8f9ce588d493b8399d7ad50599a3f] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp, totalSize=36.8 K 2024-11-14T09:14:36,461 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting f10d91d68e0c42edb1be0c92c4830d37, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1731575671008 2024-11-14T09:14:36,462 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting dd1dc19c8ca44ab4b3393fa2e72c05e4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731575671716 2024-11-14T09:14:36,462 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 93a8f9ce588d493b8399d7ad50599a3f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731575672883 2024-11-14T09:14:36,472 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a7663ca41075544279db903393226a2f] 2024-11-14T09:14:36,481 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114807f67dbabbb42a69ef3f9892c11d4cd_a7663ca41075544279db903393226a2f store=[table=TestAcidGuarantees family=A region=a7663ca41075544279db903393226a2f] 2024-11-14T09:14:36,483 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114807f67dbabbb42a69ef3f9892c11d4cd_a7663ca41075544279db903393226a2f, store=[table=TestAcidGuarantees family=A region=a7663ca41075544279db903393226a2f] 2024-11-14T09:14:36,483 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114807f67dbabbb42a69ef3f9892c11d4cd_a7663ca41075544279db903393226a2f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a7663ca41075544279db903393226a2f] 2024-11-14T09:14:36,491 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7663ca41075544279db903393226a2f#B#compaction#196 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:14:36,491 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/cb744ca943b34cdba495a99c2af26698 is 50, key is test_row_0/B:col10/1731575675046/Put/seqid=0 2024-11-14T09:14:36,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742063_1239 (size=13221) 2024-11-14T09:14:36,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742062_1238 (size=4469) 2024-11-14T09:14:36,518 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7663ca41075544279db903393226a2f#A#compaction#195 average throughput is 0.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:14:36,519 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/846935a85d424efdb867487363db3477 is 175, key is test_row_0/A:col10/1731575675046/Put/seqid=0 2024-11-14T09:14:36,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742064_1240 (size=32175) 2024-11-14T09:14:36,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-14T09:14:36,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:36,576 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing a7663ca41075544279db903393226a2f 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-14T09:14:36,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=A 2024-11-14T09:14:36,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:36,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=B 2024-11-14T09:14:36,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:36,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=C 2024-11-14T09:14:36,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:36,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111487154b7e20e347a08683334db02763c0_a7663ca41075544279db903393226a2f is 50, key is test_row_0/A:col10/1731575675088/Put/seqid=0 2024-11-14T09:14:36,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742065_1241 (size=12454) 2024-11-14T09:14:36,912 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/cb744ca943b34cdba495a99c2af26698 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/cb744ca943b34cdba495a99c2af26698 2024-11-14T09:14:36,920 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7663ca41075544279db903393226a2f/B of a7663ca41075544279db903393226a2f into cb744ca943b34cdba495a99c2af26698(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:14:36,920 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7663ca41075544279db903393226a2f: 2024-11-14T09:14:36,921 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., storeName=a7663ca41075544279db903393226a2f/B, priority=13, startTime=1731575676456; duration=0sec 2024-11-14T09:14:36,921 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:14:36,921 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7663ca41075544279db903393226a2f:B 2024-11-14T09:14:36,921 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:14:36,925 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:14:36,925 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): a7663ca41075544279db903393226a2f/C is initiating minor compaction (all files) 2024-11-14T09:14:36,926 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7663ca41075544279db903393226a2f/C in TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:36,926 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/60799e1f52a549858a75d3b20e0396f2, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/3c6b249439ed408ca32ea18ef8399635, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/206a8c04d5fd4afaa003477bda85293c] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp, totalSize=36.8 K 2024-11-14T09:14:36,927 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 60799e1f52a549858a75d3b20e0396f2, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1731575671008 2024-11-14T09:14:36,927 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c6b249439ed408ca32ea18ef8399635, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731575671716 2024-11-14T09:14:36,929 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 206a8c04d5fd4afaa003477bda85293c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731575672883 2024-11-14T09:14:36,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-14T09:14:36,941 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7663ca41075544279db903393226a2f#C#compaction#198 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:14:36,942 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/1d85f015b37f4d8c978c3ecc32ade4d9 is 50, key is test_row_0/C:col10/1731575675046/Put/seqid=0 2024-11-14T09:14:36,978 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/846935a85d424efdb867487363db3477 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/846935a85d424efdb867487363db3477 2024-11-14T09:14:36,986 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7663ca41075544279db903393226a2f/A of a7663ca41075544279db903393226a2f into 846935a85d424efdb867487363db3477(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:14:36,986 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7663ca41075544279db903393226a2f: 2024-11-14T09:14:36,986 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., storeName=a7663ca41075544279db903393226a2f/A, priority=13, startTime=1731575676454; duration=0sec 2024-11-14T09:14:36,986 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:36,986 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7663ca41075544279db903393226a2f:A 2024-11-14T09:14:36,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742066_1242 (size=13221) 2024-11-14T09:14:37,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,009 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/1d85f015b37f4d8c978c3ecc32ade4d9 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/1d85f015b37f4d8c978c3ecc32ade4d9 2024-11-14T09:14:37,014 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111487154b7e20e347a08683334db02763c0_a7663ca41075544279db903393226a2f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111487154b7e20e347a08683334db02763c0_a7663ca41075544279db903393226a2f 2024-11-14T09:14:37,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/95f6785cb2e4446782f4fb62eb914f4d, store: [table=TestAcidGuarantees family=A region=a7663ca41075544279db903393226a2f] 2024-11-14T09:14:37,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/95f6785cb2e4446782f4fb62eb914f4d is 175, key is test_row_0/A:col10/1731575675088/Put/seqid=0 2024-11-14T09:14:37,017 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7663ca41075544279db903393226a2f/C of a7663ca41075544279db903393226a2f into 1d85f015b37f4d8c978c3ecc32ade4d9(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:14:37,017 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7663ca41075544279db903393226a2f: 2024-11-14T09:14:37,017 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f., storeName=a7663ca41075544279db903393226a2f/C, priority=13, startTime=1731575676456; duration=0sec 2024-11-14T09:14:37,017 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:37,017 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7663ca41075544279db903393226a2f:C 2024-11-14T09:14:37,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742067_1243 (size=31255) 2024-11-14T09:14:37,061 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=380, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/95f6785cb2e4446782f4fb62eb914f4d 2024-11-14T09:14:37,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/ff25aa75f8df4afc9cefc8ad4eb3af4b is 50, key is test_row_0/B:col10/1731575675088/Put/seqid=0 2024-11-14T09:14:37,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742068_1244 (size=12301) 2024-11-14T09:14:37,125 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/ff25aa75f8df4afc9cefc8ad4eb3af4b 2024-11-14T09:14:37,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/a1338d762aa94068ac0506906835f430 is 50, key is test_row_0/C:col10/1731575675088/Put/seqid=0 2024-11-14T09:14:37,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742069_1245 (size=12301) 2024-11-14T09:14:37,167 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/a1338d762aa94068ac0506906835f430 2024-11-14T09:14:37,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/95f6785cb2e4446782f4fb62eb914f4d as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/95f6785cb2e4446782f4fb62eb914f4d 2024-11-14T09:14:37,185 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/95f6785cb2e4446782f4fb62eb914f4d, entries=150, sequenceid=380, filesize=30.5 K 2024-11-14T09:14:37,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/ff25aa75f8df4afc9cefc8ad4eb3af4b as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/ff25aa75f8df4afc9cefc8ad4eb3af4b 2024-11-14T09:14:37,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,192 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/ff25aa75f8df4afc9cefc8ad4eb3af4b, entries=150, sequenceid=380, filesize=12.0 K 2024-11-14T09:14:37,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/a1338d762aa94068ac0506906835f430 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/a1338d762aa94068ac0506906835f430 2024-11-14T09:14:37,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,199 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/a1338d762aa94068ac0506906835f430, entries=150, sequenceid=380, filesize=12.0 K 2024-11-14T09:14:37,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,202 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=0 B/0 for a7663ca41075544279db903393226a2f in 626ms, sequenceid=380, compaction requested=false 2024-11-14T09:14:37,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for a7663ca41075544279db903393226a2f: 2024-11-14T09:14:37,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:37,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-14T09:14:37,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-14T09:14:37,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,209 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-14T09:14:37,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,209 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4040 sec 2024-11-14T09:14:37,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,214 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 1.4160 sec 2024-11-14T09:14:37,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,356 INFO [Thread-665 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-665. 2024-11-14T09:14:37,356 DEBUG [Thread-665 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:14:37,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,356 DEBUG [Thread-665 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:14:37,357 INFO [Thread-648 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-648. 2024-11-14T09:14:37,357 DEBUG [Thread-648 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:14:37,357 DEBUG [Thread-648 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:14:37,357 DEBUG [Thread-648 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:14:37,357 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:14:37,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,359 INFO [Thread-654 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-654. 2024-11-14T09:14:37,359 DEBUG [Thread-654 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:14:37,359 DEBUG [Thread-654 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:14:37,359 DEBUG [Thread-654 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:14:37,359 DEBUG [Thread-665 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:14:37,359 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:14:37,359 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:14:37,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,367 INFO [Thread-661 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-661. 2024-11-14T09:14:37,367 DEBUG [Thread-661 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:14:37,367 DEBUG [Thread-661 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:14:37,368 DEBUG [Thread-661 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:14:37,368 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:14:37,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,381 INFO [Thread-663 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-663. 2024-11-14T09:14:37,381 DEBUG [Thread-663 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:14:37,381 DEBUG [Thread-663 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:14:37,381 DEBUG [Thread-663 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:14:37,381 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:14:37,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:37,405 INFO [Thread-659 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-659. 2024-11-14T09:14:37,405 DEBUG [Thread-659 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:14:37,405 DEBUG [Thread-659 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:14:37,405 DEBUG [Thread-659 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:14:37,405 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:14:37,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-14T09:14:37,955 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:14:40,753 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T09:14:42,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on a7663ca41075544279db903393226a2f 2024-11-14T09:14:42,458 INFO [Thread-650 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-650. 2024-11-14T09:14:42,458 INFO [Thread-652 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-652. 2024-11-14T09:14:42,458 DEBUG [Thread-650 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:14:42,458 DEBUG [Thread-650 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:14:42,458 DEBUG [Thread-652 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:14:42,458 DEBUG [Thread-652 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:14:42,458 DEBUG [Thread-650 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:14:42,458 DEBUG [Thread-652 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:14:42,458 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a7663ca41075544279db903393226a2f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T09:14:42,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=A 2024-11-14T09:14:42,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:42,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=B 2024-11-14T09:14:42,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:42,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7663ca41075544279db903393226a2f, store=C 2024-11-14T09:14:42,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:42,459 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:14:42,459 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:14:42,459 INFO [Thread-656 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-656. 2024-11-14T09:14:42,459 DEBUG [Thread-656 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:14:42,459 DEBUG [Thread-656 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:14:42,460 DEBUG [Thread-656 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:14:42,460 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:14:42,460 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T09:14:42,460 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 126 2024-11-14T09:14:42,460 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 24 2024-11-14T09:14:42,460 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 26 2024-11-14T09:14:42,460 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 97 2024-11-14T09:14:42,460 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 25 2024-11-14T09:14:42,460 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T09:14:42,460 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2819 2024-11-14T09:14:42,460 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2639 2024-11-14T09:14:42,460 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T09:14:42,460 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1172 2024-11-14T09:14:42,460 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3516 rows 2024-11-14T09:14:42,460 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1168 2024-11-14T09:14:42,460 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3499 rows 2024-11-14T09:14:42,460 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T09:14:42,460 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T09:14:42,460 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:14:42,461 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:14:42,461 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:14:42,461 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-14T09:14:42,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T09:14:42,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-14T09:14:42,465 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731575682464"}]},"ts":"1731575682464"} 2024-11-14T09:14:42,467 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T09:14:42,467 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T09:14:42,467 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T09:14:42,469 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a7663ca41075544279db903393226a2f, UNASSIGN}] 2024-11-14T09:14:42,470 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a7663ca41075544279db903393226a2f, UNASSIGN 2024-11-14T09:14:42,471 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=a7663ca41075544279db903393226a2f, regionState=CLOSING, regionLocation=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:42,472 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a7663ca41075544279db903393226a2f, UNASSIGN because future has completed 2024-11-14T09:14:42,472 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T09:14:42,473 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393}] 2024-11-14T09:14:42,480 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114cc7259ff80534ce3aa8f3ee5a8457d1d_a7663ca41075544279db903393226a2f is 50, key is test_row_0/A:col10/1731575682456/Put/seqid=0 2024-11-14T09:14:42,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742070_1246 (size=12454) 2024-11-14T09:14:42,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-14T09:14:42,625 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(122): Close a7663ca41075544279db903393226a2f 2024-11-14T09:14:42,625 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T09:14:42,625 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1722): Closing a7663ca41075544279db903393226a2f, disabling compactions & flushes 2024-11-14T09:14:42,626 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:42,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-14T09:14:42,885 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:42,891 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114cc7259ff80534ce3aa8f3ee5a8457d1d_a7663ca41075544279db903393226a2f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114cc7259ff80534ce3aa8f3ee5a8457d1d_a7663ca41075544279db903393226a2f 2024-11-14T09:14:42,892 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/b2be366c913642089ef0e1ef040a530f, store: [table=TestAcidGuarantees family=A region=a7663ca41075544279db903393226a2f] 2024-11-14T09:14:42,893 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/b2be366c913642089ef0e1ef040a530f is 175, key is test_row_0/A:col10/1731575682456/Put/seqid=0 2024-11-14T09:14:42,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742071_1247 (size=31255) 2024-11-14T09:14:42,899 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=393, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/b2be366c913642089ef0e1ef040a530f 2024-11-14T09:14:42,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/4619398a367b4803b787fe0ac5300e97 is 50, key is test_row_0/B:col10/1731575682456/Put/seqid=0 2024-11-14T09:14:42,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742072_1248 (size=12301) 2024-11-14T09:14:43,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-14T09:14:43,315 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/4619398a367b4803b787fe0ac5300e97 2024-11-14T09:14:43,323 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/31d99a9492df44e7bafdee1f3cbf232d is 50, key is test_row_0/C:col10/1731575682456/Put/seqid=0 2024-11-14T09:14:43,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742073_1249 (size=12301) 2024-11-14T09:14:43,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-14T09:14:43,729 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/31d99a9492df44e7bafdee1f3cbf232d 2024-11-14T09:14:43,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/A/b2be366c913642089ef0e1ef040a530f as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/b2be366c913642089ef0e1ef040a530f 2024-11-14T09:14:43,737 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/b2be366c913642089ef0e1ef040a530f, entries=150, sequenceid=393, filesize=30.5 K 2024-11-14T09:14:43,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/B/4619398a367b4803b787fe0ac5300e97 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/4619398a367b4803b787fe0ac5300e97 2024-11-14T09:14:43,741 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/4619398a367b4803b787fe0ac5300e97, entries=150, sequenceid=393, filesize=12.0 K 2024-11-14T09:14:43,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/.tmp/C/31d99a9492df44e7bafdee1f3cbf232d as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/31d99a9492df44e7bafdee1f3cbf232d 2024-11-14T09:14:43,746 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/31d99a9492df44e7bafdee1f3cbf232d, entries=150, sequenceid=393, filesize=12.0 K 2024-11-14T09:14:43,747 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=0 B/0 for a7663ca41075544279db903393226a2f in 1289ms, sequenceid=393, compaction requested=true 2024-11-14T09:14:43,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a7663ca41075544279db903393226a2f: 2024-11-14T09:14:43,747 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:43,747 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:43,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7663ca41075544279db903393226a2f:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:14:43,747 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. after waiting 0 ms 2024-11-14T09:14:43,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:43,748 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:43,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7663ca41075544279db903393226a2f:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:14:43,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:43,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7663ca41075544279db903393226a2f:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:14:43,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:14:43,748 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. because compaction request was cancelled 2024-11-14T09:14:43,748 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. because compaction request was cancelled 2024-11-14T09:14:43,748 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7663ca41075544279db903393226a2f:A 2024-11-14T09:14:43,748 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7663ca41075544279db903393226a2f:B 2024-11-14T09:14:43,748 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. because compaction request was cancelled 2024-11-14T09:14:43,748 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7663ca41075544279db903393226a2f:C 2024-11-14T09:14:43,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/c4ac2bcfc6ca4df294f2a8a80a44089c, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/3a8aad7792c14ec3bccd14e48f663cc1, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/d5935a348e3a4260a785f5c2be1a2683, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/257504f0297f42acb267366a3479c519, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/f99360a3e1144084aaf8eddae09b1e18, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/b790b7f0e7f44c9a9eed181d3270f72f, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/89c3b595f2ee49b98d9bf4c7c47afaa0, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/8a97df37932547978448d8d7f9ea9ca3, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/a71691244e514baabdc0f4f5c74b0383, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/748399112a2a4a62849e60e2faadb109, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/5d9bb5c1df7e482c84cc6b16f9d4b289, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/d567524433ef49ff9c84264d0811d8cf, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/d0b8fc44900d438dafe9b526ef2971f3, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/24dd8082057a4581a1fbd40574a574f2, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/7d7554db47c0437aafc12ef7aa6970ad, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/83bafa3e4c1d496c96ccf339b58d53eb, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/cd72e5a6f27e4933a562ad745f34d19c, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/efa223b6872643b4a2284d039fccff01, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/bba77d72baf847f4ba9f582c0e8f1daf, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/6dffcaedb00d4a0584089d7d3d4b85d0, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/48c982c6c21e42e892486985275fb7be, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/31315ee9288141158fc7fa1531585b0c, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/1071fa53645c499798c93eb8f80faf42, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/74d14dc748644c4aa33b67b02400b0f3, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/29620a757aac42d8b006f5b9515a0456, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/4a26183b8cae46408f9c67fd2c2e214f, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/ded35735429542b9b060de6ed7a06969] to archive 2024-11-14T09:14:43,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T09:14:43,751 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/c4ac2bcfc6ca4df294f2a8a80a44089c to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/c4ac2bcfc6ca4df294f2a8a80a44089c 2024-11-14T09:14:43,752 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/3a8aad7792c14ec3bccd14e48f663cc1 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/3a8aad7792c14ec3bccd14e48f663cc1 2024-11-14T09:14:43,754 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/d5935a348e3a4260a785f5c2be1a2683 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/d5935a348e3a4260a785f5c2be1a2683 2024-11-14T09:14:43,755 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/257504f0297f42acb267366a3479c519 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/257504f0297f42acb267366a3479c519 2024-11-14T09:14:43,756 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/f99360a3e1144084aaf8eddae09b1e18 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/f99360a3e1144084aaf8eddae09b1e18 2024-11-14T09:14:43,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/b790b7f0e7f44c9a9eed181d3270f72f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/b790b7f0e7f44c9a9eed181d3270f72f 2024-11-14T09:14:43,758 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/89c3b595f2ee49b98d9bf4c7c47afaa0 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/89c3b595f2ee49b98d9bf4c7c47afaa0 2024-11-14T09:14:43,759 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/8a97df37932547978448d8d7f9ea9ca3 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/8a97df37932547978448d8d7f9ea9ca3 2024-11-14T09:14:43,760 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/a71691244e514baabdc0f4f5c74b0383 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/a71691244e514baabdc0f4f5c74b0383 2024-11-14T09:14:43,761 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/748399112a2a4a62849e60e2faadb109 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/748399112a2a4a62849e60e2faadb109 2024-11-14T09:14:43,762 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/5d9bb5c1df7e482c84cc6b16f9d4b289 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/5d9bb5c1df7e482c84cc6b16f9d4b289 2024-11-14T09:14:43,763 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/d567524433ef49ff9c84264d0811d8cf to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/d567524433ef49ff9c84264d0811d8cf 2024-11-14T09:14:43,765 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/d0b8fc44900d438dafe9b526ef2971f3 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/d0b8fc44900d438dafe9b526ef2971f3 2024-11-14T09:14:43,766 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/24dd8082057a4581a1fbd40574a574f2 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/24dd8082057a4581a1fbd40574a574f2 2024-11-14T09:14:43,767 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/7d7554db47c0437aafc12ef7aa6970ad to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/7d7554db47c0437aafc12ef7aa6970ad 2024-11-14T09:14:43,768 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/83bafa3e4c1d496c96ccf339b58d53eb to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/83bafa3e4c1d496c96ccf339b58d53eb 2024-11-14T09:14:43,770 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/cd72e5a6f27e4933a562ad745f34d19c to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/cd72e5a6f27e4933a562ad745f34d19c 2024-11-14T09:14:43,771 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/efa223b6872643b4a2284d039fccff01 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/efa223b6872643b4a2284d039fccff01 2024-11-14T09:14:43,772 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/bba77d72baf847f4ba9f582c0e8f1daf to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/bba77d72baf847f4ba9f582c0e8f1daf 2024-11-14T09:14:43,773 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/6dffcaedb00d4a0584089d7d3d4b85d0 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/6dffcaedb00d4a0584089d7d3d4b85d0 2024-11-14T09:14:43,774 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/48c982c6c21e42e892486985275fb7be to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/48c982c6c21e42e892486985275fb7be 2024-11-14T09:14:43,776 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/31315ee9288141158fc7fa1531585b0c to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/31315ee9288141158fc7fa1531585b0c 2024-11-14T09:14:43,777 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/1071fa53645c499798c93eb8f80faf42 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/1071fa53645c499798c93eb8f80faf42 2024-11-14T09:14:43,779 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/74d14dc748644c4aa33b67b02400b0f3 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/74d14dc748644c4aa33b67b02400b0f3 2024-11-14T09:14:43,780 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/29620a757aac42d8b006f5b9515a0456 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/29620a757aac42d8b006f5b9515a0456 2024-11-14T09:14:43,781 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/4a26183b8cae46408f9c67fd2c2e214f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/4a26183b8cae46408f9c67fd2c2e214f 2024-11-14T09:14:43,782 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/ded35735429542b9b060de6ed7a06969 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/ded35735429542b9b060de6ed7a06969 2024-11-14T09:14:43,784 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/1770c2a5a27e45369fd59372e7d4d546, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/77e4e312e03d4318bd5c2df49bffaf4b, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/20edc1fd08ea483aaa89d21139a23229, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/841223c0058a41479b01cc749482c576, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/d2e6394e8c6d48cd90969bb270098478, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/012130478ed144d9a4486c7542fd1f14, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/abc4b3a7e05846b5a232533daf6eac49, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/3355906649184b49885c06eb320bc551, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/90f19a64cbc245c7b151c712018b6ee6, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/5183162510e34f2aa19ddb7335381967, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/5626061d07c9440c9f7d4ca0e5a5428b, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/6210f261404d457bba5b21ad46ac3aa9, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/e40ada958b0b467d8b0d98d7223c4e4b, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/0bcd4c2f6f6f45578fc7eb40b25de789, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/0cbb2b5e91a641049d3fc308fda76dab, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/0f7f48ff16c64443af965a9110966736, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/aa0cb24bc9134da08aae63d70dba4903, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/9ca6252dcc384ad8a4a64221e2b1eda7, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/1fefa921385b4e2a97daea82215d34a7, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/d93b9a3797884d5aaae4081143624951, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/779a5cdd1c53431eb3e83fe2b0689adc, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/5999581b11ce45a1ab5264316536b9cd, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/bb5f6df933ef47409a4847018afea7b1, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/f10d91d68e0c42edb1be0c92c4830d37, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/7fb6b2a1654b4c90a9a05be6d6b1a0fc, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/dd1dc19c8ca44ab4b3393fa2e72c05e4, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/93a8f9ce588d493b8399d7ad50599a3f] to archive 2024-11-14T09:14:43,785 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T09:14:43,786 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/1770c2a5a27e45369fd59372e7d4d546 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/1770c2a5a27e45369fd59372e7d4d546 2024-11-14T09:14:43,788 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/77e4e312e03d4318bd5c2df49bffaf4b to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/77e4e312e03d4318bd5c2df49bffaf4b 2024-11-14T09:14:43,789 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/20edc1fd08ea483aaa89d21139a23229 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/20edc1fd08ea483aaa89d21139a23229 2024-11-14T09:14:43,790 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/841223c0058a41479b01cc749482c576 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/841223c0058a41479b01cc749482c576 2024-11-14T09:14:43,792 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/d2e6394e8c6d48cd90969bb270098478 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/d2e6394e8c6d48cd90969bb270098478 2024-11-14T09:14:43,793 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/012130478ed144d9a4486c7542fd1f14 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/012130478ed144d9a4486c7542fd1f14 2024-11-14T09:14:43,794 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/abc4b3a7e05846b5a232533daf6eac49 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/abc4b3a7e05846b5a232533daf6eac49 2024-11-14T09:14:43,795 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/3355906649184b49885c06eb320bc551 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/3355906649184b49885c06eb320bc551 2024-11-14T09:14:43,796 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/90f19a64cbc245c7b151c712018b6ee6 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/90f19a64cbc245c7b151c712018b6ee6 2024-11-14T09:14:43,798 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/5183162510e34f2aa19ddb7335381967 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/5183162510e34f2aa19ddb7335381967 2024-11-14T09:14:43,799 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/5626061d07c9440c9f7d4ca0e5a5428b to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/5626061d07c9440c9f7d4ca0e5a5428b 2024-11-14T09:14:43,800 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/6210f261404d457bba5b21ad46ac3aa9 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/6210f261404d457bba5b21ad46ac3aa9 2024-11-14T09:14:43,802 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/e40ada958b0b467d8b0d98d7223c4e4b to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/e40ada958b0b467d8b0d98d7223c4e4b 2024-11-14T09:14:43,803 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/0bcd4c2f6f6f45578fc7eb40b25de789 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/0bcd4c2f6f6f45578fc7eb40b25de789 2024-11-14T09:14:43,805 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/0cbb2b5e91a641049d3fc308fda76dab to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/0cbb2b5e91a641049d3fc308fda76dab 2024-11-14T09:14:43,806 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/0f7f48ff16c64443af965a9110966736 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/0f7f48ff16c64443af965a9110966736 2024-11-14T09:14:43,808 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/aa0cb24bc9134da08aae63d70dba4903 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/aa0cb24bc9134da08aae63d70dba4903 2024-11-14T09:14:43,809 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/9ca6252dcc384ad8a4a64221e2b1eda7 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/9ca6252dcc384ad8a4a64221e2b1eda7 2024-11-14T09:14:43,815 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/1fefa921385b4e2a97daea82215d34a7 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/1fefa921385b4e2a97daea82215d34a7 2024-11-14T09:14:43,817 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/d93b9a3797884d5aaae4081143624951 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/d93b9a3797884d5aaae4081143624951 2024-11-14T09:14:43,818 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/779a5cdd1c53431eb3e83fe2b0689adc to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/779a5cdd1c53431eb3e83fe2b0689adc 2024-11-14T09:14:43,819 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/5999581b11ce45a1ab5264316536b9cd to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/5999581b11ce45a1ab5264316536b9cd 2024-11-14T09:14:43,821 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/bb5f6df933ef47409a4847018afea7b1 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/bb5f6df933ef47409a4847018afea7b1 2024-11-14T09:14:43,822 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/f10d91d68e0c42edb1be0c92c4830d37 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/f10d91d68e0c42edb1be0c92c4830d37 2024-11-14T09:14:43,823 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/7fb6b2a1654b4c90a9a05be6d6b1a0fc to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/7fb6b2a1654b4c90a9a05be6d6b1a0fc 2024-11-14T09:14:43,824 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/dd1dc19c8ca44ab4b3393fa2e72c05e4 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/dd1dc19c8ca44ab4b3393fa2e72c05e4 2024-11-14T09:14:43,825 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/93a8f9ce588d493b8399d7ad50599a3f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/93a8f9ce588d493b8399d7ad50599a3f 2024-11-14T09:14:43,827 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/60c8088514a947e6b377cf63a91933f6, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/4269b05e852c400d903e3575d7db05d9, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/d4553efcb62048789668c031a3879b08, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/2bf56dff9e3c4ad0ae2e440129be874e, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/88d919a94517449aa85c96885501043b, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/c484909f18e44f7d828ae2327e8fbcda, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/914e6f83fd684c6398cc2af0acbb8e87, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/7b3b47e54feb4f458c271ddfb1f7056a, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/27357202a8b94a9698da80b0447a8f93, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/3b6196f58d984aa6aac52c90969fdf26, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/36b5fd906bee437fb49684c3df86b592, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/b08275aba6c34e949583c53876920b5f, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/7ada07d0b2e74a46863798f7a2074a41, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/a289daf5fdd746e4a4026d539b81b8bf, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/03a90dec1814426aa0c3f54efcba964f, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/452cfb5cbfa24b23ba032263a1d7c630, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/7285a6a799374b4f81c0fe4cbda1b937, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/c501133f7ea04b4ba8696b6e05ab7e1a, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/edd99b2ad3644db6965909704a176ccc, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/00a0421d3ec1461da0ba008cb657f372, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/3d4d83b7acb34219b7fb6a977580ef9a, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/e6a1e983d07a4eba9042cf855fa705e8, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/0dc1cea199f2406ebd9f730fb563824f, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/60799e1f52a549858a75d3b20e0396f2, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/0fbc86b3a7c14691a243822406e8ff32, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/3c6b249439ed408ca32ea18ef8399635, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/206a8c04d5fd4afaa003477bda85293c] to archive 2024-11-14T09:14:43,828 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T09:14:43,830 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/60c8088514a947e6b377cf63a91933f6 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/60c8088514a947e6b377cf63a91933f6 2024-11-14T09:14:43,831 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/4269b05e852c400d903e3575d7db05d9 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/4269b05e852c400d903e3575d7db05d9 2024-11-14T09:14:43,832 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/d4553efcb62048789668c031a3879b08 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/d4553efcb62048789668c031a3879b08 2024-11-14T09:14:43,837 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/2bf56dff9e3c4ad0ae2e440129be874e to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/2bf56dff9e3c4ad0ae2e440129be874e 2024-11-14T09:14:43,838 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/88d919a94517449aa85c96885501043b to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/88d919a94517449aa85c96885501043b 2024-11-14T09:14:43,840 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/c484909f18e44f7d828ae2327e8fbcda to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/c484909f18e44f7d828ae2327e8fbcda 2024-11-14T09:14:43,841 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/914e6f83fd684c6398cc2af0acbb8e87 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/914e6f83fd684c6398cc2af0acbb8e87 2024-11-14T09:14:43,842 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/7b3b47e54feb4f458c271ddfb1f7056a to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/7b3b47e54feb4f458c271ddfb1f7056a 2024-11-14T09:14:43,843 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/27357202a8b94a9698da80b0447a8f93 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/27357202a8b94a9698da80b0447a8f93 2024-11-14T09:14:43,844 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/3b6196f58d984aa6aac52c90969fdf26 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/3b6196f58d984aa6aac52c90969fdf26 2024-11-14T09:14:43,845 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/36b5fd906bee437fb49684c3df86b592 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/36b5fd906bee437fb49684c3df86b592 2024-11-14T09:14:43,846 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/b08275aba6c34e949583c53876920b5f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/b08275aba6c34e949583c53876920b5f 2024-11-14T09:14:43,848 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/7ada07d0b2e74a46863798f7a2074a41 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/7ada07d0b2e74a46863798f7a2074a41 2024-11-14T09:14:43,849 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/a289daf5fdd746e4a4026d539b81b8bf to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/a289daf5fdd746e4a4026d539b81b8bf 2024-11-14T09:14:43,850 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/03a90dec1814426aa0c3f54efcba964f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/03a90dec1814426aa0c3f54efcba964f 2024-11-14T09:14:43,851 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/452cfb5cbfa24b23ba032263a1d7c630 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/452cfb5cbfa24b23ba032263a1d7c630 2024-11-14T09:14:43,853 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/7285a6a799374b4f81c0fe4cbda1b937 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/7285a6a799374b4f81c0fe4cbda1b937 2024-11-14T09:14:43,854 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/c501133f7ea04b4ba8696b6e05ab7e1a to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/c501133f7ea04b4ba8696b6e05ab7e1a 2024-11-14T09:14:43,855 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/edd99b2ad3644db6965909704a176ccc to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/edd99b2ad3644db6965909704a176ccc 2024-11-14T09:14:43,856 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/00a0421d3ec1461da0ba008cb657f372 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/00a0421d3ec1461da0ba008cb657f372 2024-11-14T09:14:43,858 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/3d4d83b7acb34219b7fb6a977580ef9a to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/3d4d83b7acb34219b7fb6a977580ef9a 2024-11-14T09:14:43,859 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/e6a1e983d07a4eba9042cf855fa705e8 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/e6a1e983d07a4eba9042cf855fa705e8 2024-11-14T09:14:43,860 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/0dc1cea199f2406ebd9f730fb563824f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/0dc1cea199f2406ebd9f730fb563824f 2024-11-14T09:14:43,862 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/60799e1f52a549858a75d3b20e0396f2 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/60799e1f52a549858a75d3b20e0396f2 2024-11-14T09:14:43,866 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/0fbc86b3a7c14691a243822406e8ff32 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/0fbc86b3a7c14691a243822406e8ff32 2024-11-14T09:14:43,867 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/3c6b249439ed408ca32ea18ef8399635 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/3c6b249439ed408ca32ea18ef8399635 2024-11-14T09:14:43,870 DEBUG [StoreCloser-TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/206a8c04d5fd4afaa003477bda85293c to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/206a8c04d5fd4afaa003477bda85293c 2024-11-14T09:14:43,882 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/recovered.edits/396.seqid, newMaxSeqId=396, maxSeqId=4 2024-11-14T09:14:43,883 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f. 2024-11-14T09:14:43,883 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1676): Region close journal for a7663ca41075544279db903393226a2f: Waiting for close lock at 1731575682625Running coprocessor pre-close hooks at 1731575682625Disabling compacts and flushes for region at 1731575682625Disabling writes for close at 1731575683748 (+1123 ms)Writing region close event to WAL at 1731575683874 (+126 ms)Running coprocessor post-close hooks at 1731575683883 (+9 ms)Closed at 1731575683883 2024-11-14T09:14:43,885 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(157): Closed a7663ca41075544279db903393226a2f 2024-11-14T09:14:43,886 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=a7663ca41075544279db903393226a2f, regionState=CLOSED 2024-11-14T09:14:43,888 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 because future has completed 2024-11-14T09:14:43,891 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-14T09:14:43,891 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseRegionProcedure a7663ca41075544279db903393226a2f, server=83f56b55f2af,39927,1731575622393 in 1.4160 sec 2024-11-14T09:14:43,893 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-14T09:14:43,893 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a7663ca41075544279db903393226a2f, UNASSIGN in 1.4220 sec 2024-11-14T09:14:43,896 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-14T09:14:43,896 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4270 sec 2024-11-14T09:14:43,898 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731575683897"}]},"ts":"1731575683897"} 2024-11-14T09:14:43,899 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T09:14:43,899 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T09:14:43,901 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4390 sec 2024-11-14T09:14:44,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-14T09:14:44,616 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:14:44,616 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-14T09:14:44,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:14:44,619 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:14:44,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-14T09:14:44,620 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=60, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:14:44,623 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f 2024-11-14T09:14:44,626 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A, FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B, FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C, FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/recovered.edits] 2024-11-14T09:14:44,630 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/846935a85d424efdb867487363db3477 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/846935a85d424efdb867487363db3477 2024-11-14T09:14:44,632 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/95f6785cb2e4446782f4fb62eb914f4d to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/95f6785cb2e4446782f4fb62eb914f4d 2024-11-14T09:14:44,633 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/b2be366c913642089ef0e1ef040a530f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/A/b2be366c913642089ef0e1ef040a530f 2024-11-14T09:14:44,636 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/4619398a367b4803b787fe0ac5300e97 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/4619398a367b4803b787fe0ac5300e97 2024-11-14T09:14:44,638 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/cb744ca943b34cdba495a99c2af26698 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/cb744ca943b34cdba495a99c2af26698 2024-11-14T09:14:44,639 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/ff25aa75f8df4afc9cefc8ad4eb3af4b to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/B/ff25aa75f8df4afc9cefc8ad4eb3af4b 2024-11-14T09:14:44,642 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/1d85f015b37f4d8c978c3ecc32ade4d9 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/1d85f015b37f4d8c978c3ecc32ade4d9 2024-11-14T09:14:44,644 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/31d99a9492df44e7bafdee1f3cbf232d to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/31d99a9492df44e7bafdee1f3cbf232d 2024-11-14T09:14:44,646 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/a1338d762aa94068ac0506906835f430 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/C/a1338d762aa94068ac0506906835f430 2024-11-14T09:14:44,649 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/recovered.edits/396.seqid to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f/recovered.edits/396.seqid 2024-11-14T09:14:44,650 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/a7663ca41075544279db903393226a2f 2024-11-14T09:14:44,650 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T09:14:44,651 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T09:14:44,653 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-14T09:14:44,658 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114094bce857cb243dcb3083d3a86030fca_a7663ca41075544279db903393226a2f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114094bce857cb243dcb3083d3a86030fca_a7663ca41075544279db903393226a2f 2024-11-14T09:14:44,660 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411140ac0442cbec64187820bfa31435e73df_a7663ca41075544279db903393226a2f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411140ac0442cbec64187820bfa31435e73df_a7663ca41075544279db903393226a2f 2024-11-14T09:14:44,661 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114183429e7b7c7478c8452d58ddcfd238a_a7663ca41075544279db903393226a2f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114183429e7b7c7478c8452d58ddcfd238a_a7663ca41075544279db903393226a2f 2024-11-14T09:14:44,665 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111420999d04577d438085dbc17c3f46e678_a7663ca41075544279db903393226a2f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111420999d04577d438085dbc17c3f46e678_a7663ca41075544279db903393226a2f 2024-11-14T09:14:44,667 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411142fe4a1e05cc04505b81cf18331e3b060_a7663ca41075544279db903393226a2f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411142fe4a1e05cc04505b81cf18331e3b060_a7663ca41075544279db903393226a2f 2024-11-14T09:14:44,669 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111445ad60d1ebbe4b0dac386b2c6fc51d44_a7663ca41075544279db903393226a2f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111445ad60d1ebbe4b0dac386b2c6fc51d44_a7663ca41075544279db903393226a2f 2024-11-14T09:14:44,671 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111484194a96787545728ca4405f3ddf4b70_a7663ca41075544279db903393226a2f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111484194a96787545728ca4405f3ddf4b70_a7663ca41075544279db903393226a2f 2024-11-14T09:14:44,672 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114852c6c8b32ec44fdad4e4df2826ec39f_a7663ca41075544279db903393226a2f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114852c6c8b32ec44fdad4e4df2826ec39f_a7663ca41075544279db903393226a2f 2024-11-14T09:14:44,674 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114853416af207c41ffb5d2340c811c05f1_a7663ca41075544279db903393226a2f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114853416af207c41ffb5d2340c811c05f1_a7663ca41075544279db903393226a2f 2024-11-14T09:14:44,675 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111487154b7e20e347a08683334db02763c0_a7663ca41075544279db903393226a2f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111487154b7e20e347a08683334db02763c0_a7663ca41075544279db903393226a2f 2024-11-14T09:14:44,679 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111487210816b2564dabb7520fabd5b85cf4_a7663ca41075544279db903393226a2f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111487210816b2564dabb7520fabd5b85cf4_a7663ca41075544279db903393226a2f 2024-11-14T09:14:44,681 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111492544ea31f894bbc8464da2ca81a29f3_a7663ca41075544279db903393226a2f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111492544ea31f894bbc8464da2ca81a29f3_a7663ca41075544279db903393226a2f 2024-11-14T09:14:44,682 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111497bdddf95f9f42fd9df617c97fab554d_a7663ca41075544279db903393226a2f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111497bdddf95f9f42fd9df617c97fab554d_a7663ca41075544279db903393226a2f 2024-11-14T09:14:44,684 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114a557f8b7b383465a86a26b70bdc438d9_a7663ca41075544279db903393226a2f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114a557f8b7b383465a86a26b70bdc438d9_a7663ca41075544279db903393226a2f 2024-11-14T09:14:44,685 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c322142bfe994cad805b26bf4c47eb17_a7663ca41075544279db903393226a2f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c322142bfe994cad805b26bf4c47eb17_a7663ca41075544279db903393226a2f 2024-11-14T09:14:44,686 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114cc7259ff80534ce3aa8f3ee5a8457d1d_a7663ca41075544279db903393226a2f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114cc7259ff80534ce3aa8f3ee5a8457d1d_a7663ca41075544279db903393226a2f 2024-11-14T09:14:44,688 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e0af3287ff7b40aba2a32bae31c80b82_a7663ca41075544279db903393226a2f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e0af3287ff7b40aba2a32bae31c80b82_a7663ca41075544279db903393226a2f 2024-11-14T09:14:44,689 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e44c1e6ea4fc438babc20fc0ad0c6723_a7663ca41075544279db903393226a2f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e44c1e6ea4fc438babc20fc0ad0c6723_a7663ca41075544279db903393226a2f 2024-11-14T09:14:44,690 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ee037bda70d54ac59d52662ade1d4571_a7663ca41075544279db903393226a2f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ee037bda70d54ac59d52662ade1d4571_a7663ca41075544279db903393226a2f 2024-11-14T09:14:44,692 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f37fcbf495024614a12d3ce86b2e9a7c_a7663ca41075544279db903393226a2f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f37fcbf495024614a12d3ce86b2e9a7c_a7663ca41075544279db903393226a2f 2024-11-14T09:14:44,693 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f610524d027a4ee5a642e6ea8acbaef6_a7663ca41075544279db903393226a2f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f610524d027a4ee5a642e6ea8acbaef6_a7663ca41075544279db903393226a2f 2024-11-14T09:14:44,694 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T09:14:44,696 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=60, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:14:44,699 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T09:14:44,701 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T09:14:44,703 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=60, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:14:44,703 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T09:14:44,703 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731575684703"}]},"ts":"9223372036854775807"} 2024-11-14T09:14:44,705 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T09:14:44,705 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a7663ca41075544279db903393226a2f, NAME => 'TestAcidGuarantees,,1731575654201.a7663ca41075544279db903393226a2f.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T09:14:44,705 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T09:14:44,705 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731575684705"}]},"ts":"9223372036854775807"} 2024-11-14T09:14:44,707 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T09:14:44,708 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=60, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:14:44,710 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 92 msec 2024-11-14T09:14:44,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-14T09:14:44,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T09:14:44,726 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:14:44,738 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248 (was 248), OpenFileDescriptor=453 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=475 (was 408) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7424 (was 7496) 2024-11-14T09:14:44,753 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=475, ProcessCount=11, AvailableMemoryMB=7422 2024-11-14T09:14:44,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T09:14:44,755 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T09:14:44,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T09:14:44,757 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T09:14:44,757 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:44,758 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 61 2024-11-14T09:14:44,758 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T09:14:44,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-14T09:14:44,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742074_1250 (size=1061) 2024-11-14T09:14:44,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-14T09:14:45,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-14T09:14:45,170 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db 2024-11-14T09:14:45,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742075_1251 (size=53) 2024-11-14T09:14:45,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-14T09:14:45,578 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:14:45,578 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 8664731a05c52a198640968a7aa48527, disabling compactions & flushes 2024-11-14T09:14:45,578 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:45,578 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:45,578 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. after waiting 0 ms 2024-11-14T09:14:45,578 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:45,578 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:45,578 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8664731a05c52a198640968a7aa48527: Waiting for close lock at 1731575685578Disabling compacts and flushes for region at 1731575685578Disabling writes for close at 1731575685578Writing region close event to WAL at 1731575685578Closed at 1731575685578 2024-11-14T09:14:45,580 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T09:14:45,580 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731575685580"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731575685580"}]},"ts":"1731575685580"} 2024-11-14T09:14:45,582 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T09:14:45,582 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T09:14:45,582 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731575685582"}]},"ts":"1731575685582"} 2024-11-14T09:14:45,584 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T09:14:45,584 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8664731a05c52a198640968a7aa48527, ASSIGN}] 2024-11-14T09:14:45,585 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8664731a05c52a198640968a7aa48527, ASSIGN 2024-11-14T09:14:45,586 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8664731a05c52a198640968a7aa48527, ASSIGN; state=OFFLINE, location=83f56b55f2af,39927,1731575622393; forceNewPlan=false, retain=false 2024-11-14T09:14:45,736 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=8664731a05c52a198640968a7aa48527, regionState=OPENING, regionLocation=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:45,739 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8664731a05c52a198640968a7aa48527, ASSIGN because future has completed 2024-11-14T09:14:45,739 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393}] 2024-11-14T09:14:45,895 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:45,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-14T09:14:45,895 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7752): Opening region: {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} 2024-11-14T09:14:45,895 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 8664731a05c52a198640968a7aa48527 2024-11-14T09:14:45,895 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:14:45,895 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7794): checking encryption for 8664731a05c52a198640968a7aa48527 2024-11-14T09:14:45,895 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7797): checking classloading for 8664731a05c52a198640968a7aa48527 2024-11-14T09:14:45,897 INFO [StoreOpener-8664731a05c52a198640968a7aa48527-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 8664731a05c52a198640968a7aa48527 2024-11-14T09:14:45,898 INFO [StoreOpener-8664731a05c52a198640968a7aa48527-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T09:14:45,898 INFO [StoreOpener-8664731a05c52a198640968a7aa48527-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8664731a05c52a198640968a7aa48527 columnFamilyName A 2024-11-14T09:14:45,898 DEBUG [StoreOpener-8664731a05c52a198640968a7aa48527-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:45,899 INFO [StoreOpener-8664731a05c52a198640968a7aa48527-1 {}] regionserver.HStore(327): Store=8664731a05c52a198640968a7aa48527/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:14:45,899 INFO [StoreOpener-8664731a05c52a198640968a7aa48527-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 8664731a05c52a198640968a7aa48527 2024-11-14T09:14:45,900 INFO [StoreOpener-8664731a05c52a198640968a7aa48527-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T09:14:45,900 INFO [StoreOpener-8664731a05c52a198640968a7aa48527-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8664731a05c52a198640968a7aa48527 columnFamilyName B 2024-11-14T09:14:45,900 DEBUG [StoreOpener-8664731a05c52a198640968a7aa48527-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:45,900 INFO [StoreOpener-8664731a05c52a198640968a7aa48527-1 {}] regionserver.HStore(327): Store=8664731a05c52a198640968a7aa48527/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:14:45,900 INFO [StoreOpener-8664731a05c52a198640968a7aa48527-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 8664731a05c52a198640968a7aa48527 2024-11-14T09:14:45,901 INFO [StoreOpener-8664731a05c52a198640968a7aa48527-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T09:14:45,901 INFO [StoreOpener-8664731a05c52a198640968a7aa48527-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8664731a05c52a198640968a7aa48527 columnFamilyName C 2024-11-14T09:14:45,901 DEBUG [StoreOpener-8664731a05c52a198640968a7aa48527-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:14:45,902 INFO [StoreOpener-8664731a05c52a198640968a7aa48527-1 {}] regionserver.HStore(327): Store=8664731a05c52a198640968a7aa48527/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:14:45,902 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:45,902 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1038): replaying wal for 8664731a05c52a198640968a7aa48527 2024-11-14T09:14:45,903 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527 2024-11-14T09:14:45,903 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527 2024-11-14T09:14:45,904 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1048): stopping wal replay for 8664731a05c52a198640968a7aa48527 2024-11-14T09:14:45,904 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1060): Cleaning up temporary data for 8664731a05c52a198640968a7aa48527 2024-11-14T09:14:45,904 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T09:14:45,905 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1093): writing seq id for 8664731a05c52a198640968a7aa48527 2024-11-14T09:14:45,909 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T09:14:45,909 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1114): Opened 8664731a05c52a198640968a7aa48527; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70447271, jitterRate=0.04974614083766937}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T09:14:45,909 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8664731a05c52a198640968a7aa48527 2024-11-14T09:14:45,911 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1006): Region open journal for 8664731a05c52a198640968a7aa48527: Running coprocessor pre-open hook at 1731575685896Writing region info on filesystem at 1731575685896Initializing all the Stores at 1731575685896Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731575685896Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731575685897 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731575685897Cleaning up temporary data from old regions at 1731575685904 (+7 ms)Running coprocessor post-open hooks at 1731575685909 (+5 ms)Region opened successfully at 1731575685911 (+2 ms) 2024-11-14T09:14:45,912 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., pid=63, masterSystemTime=1731575685891 2024-11-14T09:14:45,913 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:45,913 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:45,914 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=8664731a05c52a198640968a7aa48527, regionState=OPEN, openSeqNum=2, regionLocation=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:45,915 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 because future has completed 2024-11-14T09:14:45,922 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-14T09:14:45,922 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 in 178 msec 2024-11-14T09:14:45,926 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-14T09:14:45,926 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8664731a05c52a198640968a7aa48527, ASSIGN in 338 msec 2024-11-14T09:14:45,927 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T09:14:45,927 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731575685927"}]},"ts":"1731575685927"} 2024-11-14T09:14:45,928 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T09:14:45,929 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T09:14:45,931 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1740 sec 2024-11-14T09:14:46,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-14T09:14:46,915 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:14:46,916 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@781611af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:46,916 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:14:46,916 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:14:46,917 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:14:46,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:14:46,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:14:46,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68c1342a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:46,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:14:46,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:14:46,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:14:46,919 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53530, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:14:46,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@185c00db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:46,920 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:14:46,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:14:46,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:14:46,922 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58112, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:14:46,922 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T09:14:46,923 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 83f56b55f2af,45325,1731575621660 2024-11-14T09:14:46,923 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@218c79d0 2024-11-14T09:14:46,923 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T09:14:46,924 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53534, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T09:14:46,925 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32f1f4e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:46,925 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:14:46,925 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:14:46,926 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:14:46,926 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:14:46,926 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:14:46,926 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@267cebb4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:46,926 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:14:46,926 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:14:46,926 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:14:46,927 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53552, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:14:46,927 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7187dfe3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:46,928 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cf99f0a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:46,928 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:14:46,928 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:14:46,928 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:14:46,929 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:14:46,929 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:14:46,929 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79853070, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:46,929 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:14:46,929 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:14:46,929 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:14:46,930 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53576, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:14:46,930 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52723e36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:46,930 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6594bb9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:46,931 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:14:46,931 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:14:46,931 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:14:46,931 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:14:46,931 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:14:46,931 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@359eb02c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:46,931 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:14:46,932 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:14:46,932 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:14:46,932 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53600, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:14:46,933 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e89c59c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:46,933 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61deb417, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:46,933 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:14:46,933 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:14:46,934 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:14:46,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:14:46,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:14:46,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a236b44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:46,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:14:46,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:14:46,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:14:46,935 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53628, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:14:46,935 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45f6c2df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:46,936 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47f17e9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:46,936 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:14:46,936 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:14:46,937 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:14:46,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:14:46,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:14:46,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30b2c4cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:46,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:14:46,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:14:46,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:14:46,938 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53642, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:14:46,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31ac0027, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:46,939 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7acbee0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:46,939 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:14:46,939 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:14:46,939 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:14:46,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:14:46,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:14:46,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ee14a16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:46,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:14:46,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:14:46,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:14:46,940 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53662, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:14:46,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35a92842, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:46,941 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46eaeeea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:46,941 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:14:46,941 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:14:46,942 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:14:46,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:14:46,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:14:46,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fff4889, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:46,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:14:46,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:14:46,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:14:46,943 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53670, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:14:46,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@528ae5de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:46,944 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bee866e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:46,944 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:14:46,944 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:14:46,945 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:14:46,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:14:46,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:14:46,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a1d808, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:46,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:14:46,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:14:46,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:14:46,946 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53692, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:14:46,946 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e4ca728, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:46,947 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@148ac527, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:46,947 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:14:46,947 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:14:46,947 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:14:46,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:14:46,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:14:46,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11a1b17e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:46,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:14:46,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:14:46,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:14:46,949 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53714, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:14:46,949 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a3e8c12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:46,949 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4199deaf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:46,949 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:14:46,949 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:14:46,950 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:14:46,950 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:14:46,950 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:14:46,950 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@511df334, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:46,950 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:14:46,950 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:14:46,950 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:14:46,951 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53744, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:14:46,952 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30631f0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:14:46,952 DEBUG [Thread-1172 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:14:46,952 DEBUG [Thread-1166 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:14:46,952 DEBUG [Thread-1183 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:14:46,952 DEBUG [Thread-1168 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:14:46,953 DEBUG [Thread-1181 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:14:46,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:14:46,953 DEBUG [Thread-1179 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:14:46,953 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:14:46,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:14:46,953 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:14:46,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:14:46,954 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:14:46,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:14:46,954 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:14:46,954 DEBUG [Thread-1164 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:14:46,954 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:14:46,954 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:14:46,954 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:14:46,955 DEBUG [Thread-1170 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:14:46,955 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58122, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:14:46,955 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58140, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:14:46,955 DEBUG [Thread-1177 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:14:46,955 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58120, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:14:46,955 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:14:46,955 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58158, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:14:46,955 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:14:46,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:14:46,955 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:14:46,955 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58142, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:14:46,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:14:46,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:14:46,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-14T09:14:46,957 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58184, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:14:46,957 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58168, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:14:46,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-14T09:14:46,958 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58194, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:14:46,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2] 2024-11-14T09:14:46,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2] 2024-11-14T09:14:46,958 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:14:46,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2] 2024-11-14T09:14:46,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2] 2024-11-14T09:14:46,959 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:14:46,959 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:14:46,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2] 2024-11-14T09:14:46,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2] 2024-11-14T09:14:46,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:14:46,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2] 2024-11-14T09:14:46,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2] 2024-11-14T09:14:46,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:14:46,963 DEBUG [Thread-1175 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:14:46,963 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58200, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:14:46,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:14:46,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:14:46,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2] 2024-11-14T09:14:46,966 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58202, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:14:46,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2] 2024-11-14T09:14:46,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on 8664731a05c52a198640968a7aa48527 2024-11-14T09:14:46,970 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8664731a05c52a198640968a7aa48527 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T09:14:46,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=A 2024-11-14T09:14:46,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:46,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=B 2024-11-14T09:14:46,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:46,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=C 2024-11-14T09:14:46,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:47,008 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/d13de21550fd464caced9b80ccc430e3 is 50, key is test_row_0/A:col10/1731575686970/Put/seqid=0 2024-11-14T09:14:47,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:47,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58122 deadline: 1731575747005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:47,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:47,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:47,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:47,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:47,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58184 deadline: 1731575747006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:47,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:47,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:47,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:47,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:47,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58142 deadline: 1731575747008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:47,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:47,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:47,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:47,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:47,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58168 deadline: 1731575747012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:47,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:47,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:47,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:47,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:47,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731575747014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:47,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:47,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:47,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:47,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742076_1252 (size=16681) 2024-11-14T09:14:47,046 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/d13de21550fd464caced9b80ccc430e3 2024-11-14T09:14:47,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-14T09:14:47,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/a40146e7efc94717bb927bf0f3699bfb is 50, key is test_row_0/B:col10/1731575686970/Put/seqid=0 2024-11-14T09:14:47,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742077_1253 (size=12001) 2024-11-14T09:14:47,097 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/a40146e7efc94717bb927bf0f3699bfb 2024-11-14T09:14:47,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-14T09:14:47,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:47,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. as already flushing 2024-11-14T09:14:47,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:47,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:47,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:47,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:47,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:47,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58122 deadline: 1731575747125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:47,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:47,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58184 deadline: 1731575747125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:47,129 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:47,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:47,129 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:47,129 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:47,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58142 deadline: 1731575747126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:47,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:47,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:47,129 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:47,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:47,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:47,129 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:47,129 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:47,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58168 deadline: 1731575747126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:47,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:47,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:47,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:47,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:47,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731575747130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:47,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:47,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:47,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:47,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/de0b291128bf4494bb95db285538a65e is 50, key is test_row_0/C:col10/1731575686970/Put/seqid=0 2024-11-14T09:14:47,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742078_1254 (size=12001) 2024-11-14T09:14:47,175 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/de0b291128bf4494bb95db285538a65e 2024-11-14T09:14:47,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/d13de21550fd464caced9b80ccc430e3 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/d13de21550fd464caced9b80ccc430e3 2024-11-14T09:14:47,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/d13de21550fd464caced9b80ccc430e3, entries=250, sequenceid=14, filesize=16.3 K 2024-11-14T09:14:47,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/a40146e7efc94717bb927bf0f3699bfb as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/a40146e7efc94717bb927bf0f3699bfb 2024-11-14T09:14:47,201 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/a40146e7efc94717bb927bf0f3699bfb, entries=150, sequenceid=14, filesize=11.7 K 2024-11-14T09:14:47,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/de0b291128bf4494bb95db285538a65e as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/de0b291128bf4494bb95db285538a65e 2024-11-14T09:14:47,207 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/de0b291128bf4494bb95db285538a65e, entries=150, sequenceid=14, filesize=11.7 K 2024-11-14T09:14:47,209 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 8664731a05c52a198640968a7aa48527 in 239ms, sequenceid=14, compaction requested=false 2024-11-14T09:14:47,209 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-14T09:14:47,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8664731a05c52a198640968a7aa48527: 2024-11-14T09:14:47,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-14T09:14:47,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:47,271 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 8664731a05c52a198640968a7aa48527 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-14T09:14:47,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=A 2024-11-14T09:14:47,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:47,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=B 2024-11-14T09:14:47,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:47,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=C 2024-11-14T09:14:47,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:47,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-14T09:14:47,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/4792663d6d1246f880d9ddabff255221 is 50, key is test_row_0/A:col10/1731575687002/Put/seqid=0 2024-11-14T09:14:47,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742079_1255 (size=12001) 2024-11-14T09:14:47,298 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/4792663d6d1246f880d9ddabff255221 2024-11-14T09:14:47,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/ad323a1546aa4d969ae3dd3b5b7dbf2a is 50, key is test_row_0/B:col10/1731575687002/Put/seqid=0 2024-11-14T09:14:47,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. as already flushing 2024-11-14T09:14:47,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on 8664731a05c52a198640968a7aa48527 2024-11-14T09:14:47,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742080_1256 (size=12001) 2024-11-14T09:14:47,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4773) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4699) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5194) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5188) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5184) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3333) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:47,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:47,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:47,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58122 deadline: 1731575747378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:47,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58184 deadline: 1731575747379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:47,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731575747380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:47,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4773) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4699) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5194) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5188) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5184) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3333) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:47,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4773) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4699) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5194) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5188) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5184) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3333) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:47,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:47,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:47,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:47,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:47,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:47,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:47,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:47,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:47,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58142 deadline: 1731575747386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:47,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:47,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:47,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:47,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58168 deadline: 1731575747387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:47,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:47,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:47,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:47,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:47,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:47,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58122 deadline: 1731575747495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:47,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:47,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:47,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:47,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:47,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731575747495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:47,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:47,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:47,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:47,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58184 deadline: 1731575747495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:47,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:47,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:47,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58168 deadline: 1731575747495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:47,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:47,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58142 deadline: 1731575747495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:47,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:47,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:47,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:47,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:47,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:47,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:47,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:47,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:47,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:47,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-14T09:14:47,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:47,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58168 deadline: 1731575747705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:47,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:47,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58184 deadline: 1731575747705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:47,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:47,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:47,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:47,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:47,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:47,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:47,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58122 deadline: 1731575747706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:47,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:47,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:47,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:47,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:47,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:47,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731575747706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:47,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:47,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58142 deadline: 1731575747706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:47,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:47,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:47,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:47,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:47,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:47,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:47,760 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/ad323a1546aa4d969ae3dd3b5b7dbf2a 2024-11-14T09:14:47,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/6528463614fd46b48a015541900dac89 is 50, key is test_row_0/C:col10/1731575687002/Put/seqid=0 2024-11-14T09:14:47,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742081_1257 (size=12001) 2024-11-14T09:14:48,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:48,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58122 deadline: 1731575748015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:48,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:48,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:48,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:48,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:48,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58142 deadline: 1731575748015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:48,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:48,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:48,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:48,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58184 deadline: 1731575748015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:48,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:48,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:48,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:48,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58168 deadline: 1731575748016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:48,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:48,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:48,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:48,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:48,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731575748016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:48,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:48,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:48,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:48,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:48,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:48,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-14T09:14:48,180 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/6528463614fd46b48a015541900dac89 2024-11-14T09:14:48,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/4792663d6d1246f880d9ddabff255221 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/4792663d6d1246f880d9ddabff255221 2024-11-14T09:14:48,199 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/4792663d6d1246f880d9ddabff255221, entries=150, sequenceid=33, filesize=11.7 K 2024-11-14T09:14:48,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/ad323a1546aa4d969ae3dd3b5b7dbf2a as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/ad323a1546aa4d969ae3dd3b5b7dbf2a 2024-11-14T09:14:48,206 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/ad323a1546aa4d969ae3dd3b5b7dbf2a, entries=150, sequenceid=33, filesize=11.7 K 2024-11-14T09:14:48,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/6528463614fd46b48a015541900dac89 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/6528463614fd46b48a015541900dac89 2024-11-14T09:14:48,212 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/6528463614fd46b48a015541900dac89, entries=150, sequenceid=33, filesize=11.7 K 2024-11-14T09:14:48,213 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 8664731a05c52a198640968a7aa48527 in 942ms, sequenceid=33, compaction requested=false 2024-11-14T09:14:48,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 8664731a05c52a198640968a7aa48527: 2024-11-14T09:14:48,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:48,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-14T09:14:48,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-14T09:14:48,218 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-14T09:14:48,218 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2560 sec 2024-11-14T09:14:48,220 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.2640 sec 2024-11-14T09:14:48,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on 8664731a05c52a198640968a7aa48527 2024-11-14T09:14:48,528 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8664731a05c52a198640968a7aa48527 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-14T09:14:48,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=A 2024-11-14T09:14:48,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:48,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=B 2024-11-14T09:14:48,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:48,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=C 2024-11-14T09:14:48,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:48,536 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/e41821b919cf41bcb5757ec0e52eaf8d is 50, key is test_row_0/A:col10/1731575687385/Put/seqid=0 2024-11-14T09:14:48,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742082_1258 (size=12001) 2024-11-14T09:14:48,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:48,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:48,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731575748545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:48,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58168 deadline: 1731575748546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:48,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:48,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:48,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:48,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:48,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:48,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:48,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:48,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58142 deadline: 1731575748549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:48,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:48,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:48,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:48,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:48,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58184 deadline: 1731575748549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:48,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:48,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:48,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:48,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:48,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58122 deadline: 1731575748551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:48,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:48,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:48,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:48,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:48,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731575748655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:48,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:48,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:48,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:48,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58168 deadline: 1731575748655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:48,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:48,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:48,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:48,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:48,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:48,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58142 deadline: 1731575748656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:48,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:48,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:48,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:48,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:48,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58184 deadline: 1731575748656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:48,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:48,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:48,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:48,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:48,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58122 deadline: 1731575748665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:48,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:48,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:48,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:48,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:48,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731575748864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:48,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:48,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58168 deadline: 1731575748865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:48,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:48,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:48,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:48,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:48,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:48,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:48,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58142 deadline: 1731575748865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:48,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:48,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:48,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:48,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:48,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58184 deadline: 1731575748865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:48,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:48,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:48,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:48,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:48,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:48,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58122 deadline: 1731575748875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:48,877 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:48,877 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:48,877 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:48,943 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/e41821b919cf41bcb5757ec0e52eaf8d 2024-11-14T09:14:48,951 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/e0104697d5404e9d94ad3b0f1158c489 is 50, key is test_row_0/B:col10/1731575687385/Put/seqid=0 2024-11-14T09:14:48,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742083_1259 (size=12001) 2024-11-14T09:14:48,968 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/e0104697d5404e9d94ad3b0f1158c489 2024-11-14T09:14:49,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/2f0a82a8d0cb42f2bc81ac403b989b71 is 50, key is test_row_0/C:col10/1731575687385/Put/seqid=0 2024-11-14T09:14:49,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742084_1260 (size=12001) 2024-11-14T09:14:49,023 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/2f0a82a8d0cb42f2bc81ac403b989b71 2024-11-14T09:14:49,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/e41821b919cf41bcb5757ec0e52eaf8d as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/e41821b919cf41bcb5757ec0e52eaf8d 2024-11-14T09:14:49,040 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/e41821b919cf41bcb5757ec0e52eaf8d, entries=150, sequenceid=49, filesize=11.7 K 2024-11-14T09:14:49,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/e0104697d5404e9d94ad3b0f1158c489 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/e0104697d5404e9d94ad3b0f1158c489 2024-11-14T09:14:49,047 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/e0104697d5404e9d94ad3b0f1158c489, entries=150, sequenceid=49, filesize=11.7 K 2024-11-14T09:14:49,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/2f0a82a8d0cb42f2bc81ac403b989b71 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/2f0a82a8d0cb42f2bc81ac403b989b71 2024-11-14T09:14:49,053 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/2f0a82a8d0cb42f2bc81ac403b989b71, entries=150, sequenceid=49, filesize=11.7 K 2024-11-14T09:14:49,054 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 8664731a05c52a198640968a7aa48527 in 526ms, sequenceid=49, compaction requested=true 2024-11-14T09:14:49,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8664731a05c52a198640968a7aa48527: 2024-11-14T09:14:49,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8664731a05c52a198640968a7aa48527:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:14:49,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:49,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8664731a05c52a198640968a7aa48527:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:14:49,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:49,055 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:14:49,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8664731a05c52a198640968a7aa48527:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:14:49,055 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:14:49,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:14:49,056 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:14:49,056 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:14:49,056 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): 8664731a05c52a198640968a7aa48527/B is initiating minor compaction (all files) 2024-11-14T09:14:49,056 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): 8664731a05c52a198640968a7aa48527/A is initiating minor compaction (all files) 2024-11-14T09:14:49,056 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8664731a05c52a198640968a7aa48527/B in TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:49,056 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8664731a05c52a198640968a7aa48527/A in TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:49,056 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/a40146e7efc94717bb927bf0f3699bfb, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/ad323a1546aa4d969ae3dd3b5b7dbf2a, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/e0104697d5404e9d94ad3b0f1158c489] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp, totalSize=35.2 K 2024-11-14T09:14:49,056 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/d13de21550fd464caced9b80ccc430e3, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/4792663d6d1246f880d9ddabff255221, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/e41821b919cf41bcb5757ec0e52eaf8d] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp, totalSize=39.7 K 2024-11-14T09:14:49,057 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting d13de21550fd464caced9b80ccc430e3, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731575686966 2024-11-14T09:14:49,057 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting a40146e7efc94717bb927bf0f3699bfb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731575686968 2024-11-14T09:14:49,057 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4792663d6d1246f880d9ddabff255221, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731575687002 2024-11-14T09:14:49,057 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting ad323a1546aa4d969ae3dd3b5b7dbf2a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731575687002 2024-11-14T09:14:49,058 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting e41821b919cf41bcb5757ec0e52eaf8d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731575687385 2024-11-14T09:14:49,058 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting e0104697d5404e9d94ad3b0f1158c489, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731575687385 2024-11-14T09:14:49,086 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8664731a05c52a198640968a7aa48527#A#compaction#213 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:14:49,086 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/efb9fa9402e741588dc5f0a8cacd4c1d is 50, key is test_row_0/A:col10/1731575687385/Put/seqid=0 2024-11-14T09:14:49,088 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8664731a05c52a198640968a7aa48527#B#compaction#214 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:14:49,089 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/4daad9fc31564870ba30f63a88de12a5 is 50, key is test_row_0/B:col10/1731575687385/Put/seqid=0 2024-11-14T09:14:49,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-14T09:14:49,105 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:14:49,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742085_1261 (size=12104) 2024-11-14T09:14:49,105 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:14:49,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-14T09:14:49,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-14T09:14:49,109 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:14:49,110 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:14:49,110 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:14:49,116 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/4daad9fc31564870ba30f63a88de12a5 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/4daad9fc31564870ba30f63a88de12a5 2024-11-14T09:14:49,122 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8664731a05c52a198640968a7aa48527/B of 8664731a05c52a198640968a7aa48527 into 4daad9fc31564870ba30f63a88de12a5(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:14:49,122 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8664731a05c52a198640968a7aa48527: 2024-11-14T09:14:49,122 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., storeName=8664731a05c52a198640968a7aa48527/B, priority=13, startTime=1731575689055; duration=0sec 2024-11-14T09:14:49,122 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:14:49,122 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8664731a05c52a198640968a7aa48527:B 2024-11-14T09:14:49,122 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:14:49,124 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:14:49,124 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): 8664731a05c52a198640968a7aa48527/C is initiating minor compaction (all files) 2024-11-14T09:14:49,124 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8664731a05c52a198640968a7aa48527/C in TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:49,124 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/de0b291128bf4494bb95db285538a65e, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/6528463614fd46b48a015541900dac89, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/2f0a82a8d0cb42f2bc81ac403b989b71] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp, totalSize=35.2 K 2024-11-14T09:14:49,125 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting de0b291128bf4494bb95db285538a65e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731575686968 2024-11-14T09:14:49,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742086_1262 (size=12104) 2024-11-14T09:14:49,125 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 6528463614fd46b48a015541900dac89, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731575687002 2024-11-14T09:14:49,126 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f0a82a8d0cb42f2bc81ac403b989b71, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731575687385 2024-11-14T09:14:49,139 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8664731a05c52a198640968a7aa48527#C#compaction#215 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:14:49,140 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/c6d0aac053af4eabbab58711b99753e6 is 50, key is test_row_0/C:col10/1731575687385/Put/seqid=0 2024-11-14T09:14:49,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742087_1263 (size=12104) 2024-11-14T09:14:49,169 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/c6d0aac053af4eabbab58711b99753e6 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/c6d0aac053af4eabbab58711b99753e6 2024-11-14T09:14:49,174 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8664731a05c52a198640968a7aa48527/C of 8664731a05c52a198640968a7aa48527 into c6d0aac053af4eabbab58711b99753e6(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:14:49,175 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8664731a05c52a198640968a7aa48527: 2024-11-14T09:14:49,175 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., storeName=8664731a05c52a198640968a7aa48527/C, priority=13, startTime=1731575689055; duration=0sec 2024-11-14T09:14:49,175 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:49,175 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8664731a05c52a198640968a7aa48527:C 2024-11-14T09:14:49,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on 8664731a05c52a198640968a7aa48527 2024-11-14T09:14:49,181 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8664731a05c52a198640968a7aa48527 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-14T09:14:49,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=A 2024-11-14T09:14:49,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:49,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=B 2024-11-14T09:14:49,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:49,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=C 2024-11-14T09:14:49,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:49,192 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/aa359d9fcbf54b9ea0aa546cf88c653e is 50, key is test_row_0/A:col10/1731575688543/Put/seqid=0 2024-11-14T09:14:49,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742088_1264 (size=12001) 2024-11-14T09:14:49,200 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/aa359d9fcbf54b9ea0aa546cf88c653e 2024-11-14T09:14:49,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:49,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58122 deadline: 1731575749194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:49,205 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:49,205 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:49,205 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:49,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:49,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58184 deadline: 1731575749200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:49,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:49,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:49,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:49,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58142 deadline: 1731575749202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:49,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:49,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:49,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:49,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731575749204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:49,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:49,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:49,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:49,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58168 deadline: 1731575749205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:49,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:49,208 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:49,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:49,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:49,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:49,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:49,213 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/eb8ae91782014f808a9d4c8efac1e7cb is 50, key is test_row_0/B:col10/1731575688543/Put/seqid=0 2024-11-14T09:14:49,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-14T09:14:49,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742089_1265 (size=12001) 2024-11-14T09:14:49,263 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-14T09:14:49,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:49,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. as already flushing 2024-11-14T09:14:49,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:49,263 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:49,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:49,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:49,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:49,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58122 deadline: 1731575749314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:49,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:49,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:49,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58168 deadline: 1731575749315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:49,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:49,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:49,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:49,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:49,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:49,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58142 deadline: 1731575749315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:49,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:49,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:49,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:49,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:49,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:49,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731575749315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:49,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:49,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:49,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:49,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:49,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58184 deadline: 1731575749315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:49,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:49,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:49,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:49,411 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T09:14:49,416 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-14T09:14:49,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:49,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. as already flushing 2024-11-14T09:14:49,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:49,417 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:49,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:49,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:49,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-14T09:14:49,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:49,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58168 deadline: 1731575749525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:49,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:49,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:49,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:49,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:49,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58142 deadline: 1731575749525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:49,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:49,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731575749526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:49,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:49,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:49,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58184 deadline: 1731575749526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:49,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:49,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:49,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:49,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58122 deadline: 1731575749526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:49,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:49,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:49,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:49,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:49,530 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:49,530 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:49,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:49,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:49,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:49,532 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/efb9fa9402e741588dc5f0a8cacd4c1d as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/efb9fa9402e741588dc5f0a8cacd4c1d 2024-11-14T09:14:49,538 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8664731a05c52a198640968a7aa48527/A of 8664731a05c52a198640968a7aa48527 into efb9fa9402e741588dc5f0a8cacd4c1d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:14:49,538 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8664731a05c52a198640968a7aa48527: 2024-11-14T09:14:49,538 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., storeName=8664731a05c52a198640968a7aa48527/A, priority=13, startTime=1731575689055; duration=0sec 2024-11-14T09:14:49,538 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:49,538 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8664731a05c52a198640968a7aa48527:A 2024-11-14T09:14:49,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-14T09:14:49,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:49,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. as already flushing 2024-11-14T09:14:49,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:49,570 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:49,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:49,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:49,627 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/eb8ae91782014f808a9d4c8efac1e7cb 2024-11-14T09:14:49,642 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/2cff30a1149e4424b52e3b50c527fa94 is 50, key is test_row_0/C:col10/1731575688543/Put/seqid=0 2024-11-14T09:14:49,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742090_1266 (size=12001) 2024-11-14T09:14:49,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-14T09:14:49,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:49,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. as already flushing 2024-11-14T09:14:49,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:49,725 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:49,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:49,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:49,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-14T09:14:49,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:49,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:49,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58142 deadline: 1731575749835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:49,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58168 deadline: 1731575749834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:49,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:49,838 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:49,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:49,838 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:49,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58184 deadline: 1731575749835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:49,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:49,838 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:49,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:49,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:49,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731575749835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:49,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:49,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:49,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:49,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58122 deadline: 1731575749835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:49,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:49,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:49,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:49,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:49,839 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:49,839 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:49,839 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:49,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-14T09:14:49,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:49,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. as already flushing 2024-11-14T09:14:49,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:49,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:49,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:49,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:50,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-14T09:14:50,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:50,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. as already flushing 2024-11-14T09:14:50,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:50,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:50,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:50,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:50,078 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/2cff30a1149e4424b52e3b50c527fa94 2024-11-14T09:14:50,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/aa359d9fcbf54b9ea0aa546cf88c653e as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/aa359d9fcbf54b9ea0aa546cf88c653e 2024-11-14T09:14:50,094 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/aa359d9fcbf54b9ea0aa546cf88c653e, entries=150, sequenceid=70, filesize=11.7 K 2024-11-14T09:14:50,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/eb8ae91782014f808a9d4c8efac1e7cb as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/eb8ae91782014f808a9d4c8efac1e7cb 2024-11-14T09:14:50,108 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/eb8ae91782014f808a9d4c8efac1e7cb, entries=150, sequenceid=70, filesize=11.7 K 2024-11-14T09:14:50,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/2cff30a1149e4424b52e3b50c527fa94 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/2cff30a1149e4424b52e3b50c527fa94 2024-11-14T09:14:50,117 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/2cff30a1149e4424b52e3b50c527fa94, entries=150, sequenceid=70, filesize=11.7 K 2024-11-14T09:14:50,118 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 8664731a05c52a198640968a7aa48527 in 938ms, sequenceid=70, compaction requested=false 2024-11-14T09:14:50,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8664731a05c52a198640968a7aa48527: 2024-11-14T09:14:50,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-14T09:14:50,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:50,199 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 8664731a05c52a198640968a7aa48527 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-14T09:14:50,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=A 2024-11-14T09:14:50,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:50,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=B 2024-11-14T09:14:50,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:50,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=C 2024-11-14T09:14:50,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:50,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/4892896493724401aa747e870dfe2be2 is 50, key is test_row_0/A:col10/1731575689187/Put/seqid=0 2024-11-14T09:14:50,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742091_1267 (size=12001) 2024-11-14T09:14:50,240 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/4892896493724401aa747e870dfe2be2 2024-11-14T09:14:50,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-14T09:14:50,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/af396144976f420d857994aa08f861a5 is 50, key is test_row_0/B:col10/1731575689187/Put/seqid=0 2024-11-14T09:14:50,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742092_1268 (size=12001) 2024-11-14T09:14:50,260 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/af396144976f420d857994aa08f861a5 2024-11-14T09:14:50,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/a63b0c8362ea424da1edae51e3efccf7 is 50, key is test_row_0/C:col10/1731575689187/Put/seqid=0 2024-11-14T09:14:50,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742093_1269 (size=12001) 2024-11-14T09:14:50,287 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/a63b0c8362ea424da1edae51e3efccf7 2024-11-14T09:14:50,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/4892896493724401aa747e870dfe2be2 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/4892896493724401aa747e870dfe2be2 2024-11-14T09:14:50,297 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/4892896493724401aa747e870dfe2be2, entries=150, sequenceid=84, filesize=11.7 K 2024-11-14T09:14:50,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/af396144976f420d857994aa08f861a5 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/af396144976f420d857994aa08f861a5 2024-11-14T09:14:50,308 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/af396144976f420d857994aa08f861a5, entries=150, sequenceid=84, filesize=11.7 K 2024-11-14T09:14:50,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/a63b0c8362ea424da1edae51e3efccf7 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/a63b0c8362ea424da1edae51e3efccf7 2024-11-14T09:14:50,318 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/a63b0c8362ea424da1edae51e3efccf7, entries=150, sequenceid=84, filesize=11.7 K 2024-11-14T09:14:50,319 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=0 B/0 for 8664731a05c52a198640968a7aa48527 in 119ms, sequenceid=84, compaction requested=true 2024-11-14T09:14:50,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 8664731a05c52a198640968a7aa48527: 2024-11-14T09:14:50,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:50,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-14T09:14:50,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-14T09:14:50,326 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-14T09:14:50,327 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2110 sec 2024-11-14T09:14:50,330 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.2210 sec 2024-11-14T09:14:50,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on 8664731a05c52a198640968a7aa48527 2024-11-14T09:14:50,356 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8664731a05c52a198640968a7aa48527 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T09:14:50,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=A 2024-11-14T09:14:50,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:50,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=B 2024-11-14T09:14:50,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:50,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=C 2024-11-14T09:14:50,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:50,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/2df63f3e815143f5acd4e65b88ca88ec is 50, key is test_row_0/A:col10/1731575690354/Put/seqid=0 2024-11-14T09:14:50,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742094_1270 (size=16681) 2024-11-14T09:14:50,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:50,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731575750383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:50,389 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:50,389 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:50,389 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:50,390 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731575690377, max timestamp=1731575690386Num uniques 859; 2024-11-14T09:14:50,390 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731575690377, max timestamp=1731575690386Num uniques 859; 2024-11-14T09:14:50,390 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T09:14:50,390 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T09:14:50,390 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731575690377, max timestamp=1731575690386Num uniques 859; 2024-11-14T09:14:50,390 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T09:14:50,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:50,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58168 deadline: 1731575750413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:50,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:50,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58142 deadline: 1731575750415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:50,423 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:50,423 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:50,423 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:50,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:50,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:50,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:50,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:50,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58184 deadline: 1731575750419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:50,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:50,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:50,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:50,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:50,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58122 deadline: 1731575750422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:50,430 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:50,430 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:50,430 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:50,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:50,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731575750494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:50,500 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:50,500 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:50,500 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:50,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:50,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58142 deadline: 1731575750526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:50,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:50,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:50,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:50,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:50,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58168 deadline: 1731575750536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:50,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:50,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58184 deadline: 1731575750536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:50,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:50,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:50,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:50,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:50,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58122 deadline: 1731575750536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:50,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:50,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:50,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:50,542 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:50,542 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:50,543 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:50,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:50,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731575750705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:50,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:50,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:50,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:50,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:50,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58142 deadline: 1731575750735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:50,738 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:50,738 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:50,738 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:50,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:50,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58168 deadline: 1731575750744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:50,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:50,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:50,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:50,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:50,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58184 deadline: 1731575750744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:50,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:50,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:50,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:50,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:50,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58122 deadline: 1731575750745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:50,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:50,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:50,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:50,787 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/2df63f3e815143f5acd4e65b88ca88ec 2024-11-14T09:14:50,795 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/b1f84d82d7984ea7ba859b76c1504abc is 50, key is test_row_0/B:col10/1731575690354/Put/seqid=0 2024-11-14T09:14:50,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742095_1271 (size=12001) 2024-11-14T09:14:51,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:51,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731575751015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:51,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:51,019 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:51,019 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:51,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:51,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58142 deadline: 1731575751045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:51,049 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:51,049 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:51,049 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:51,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:51,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58168 deadline: 1731575751055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:51,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:51,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58184 deadline: 1731575751055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:51,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:51,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:51,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:51,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:51,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:51,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:51,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:51,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58122 deadline: 1731575751055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:51,061 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:51,061 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:51,061 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:51,200 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/b1f84d82d7984ea7ba859b76c1504abc 2024-11-14T09:14:51,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/0835a71643f944cea5b3c875ea880c16 is 50, key is test_row_0/C:col10/1731575690354/Put/seqid=0 2024-11-14T09:14:51,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742096_1272 (size=12001) 2024-11-14T09:14:51,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-14T09:14:51,255 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:14:51,255 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:14:51,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-14T09:14:51,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-14T09:14:51,257 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:14:51,258 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:14:51,258 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:14:51,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-14T09:14:51,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-14T09:14:51,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:51,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. as already flushing 2024-11-14T09:14:51,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:51,411 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:51,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:51,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:51,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:51,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731575751525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:51,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:51,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:51,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:51,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-14T09:14:51,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:51,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. as already flushing 2024-11-14T09:14:51,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:51,564 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:51,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:51,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:51,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:51,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58142 deadline: 1731575751564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:51,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:51,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:51,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:51,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:51,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58184 deadline: 1731575751565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:51,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:51,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:51,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:51,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-14T09:14:51,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:51,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58168 deadline: 1731575751575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:51,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:51,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58122 deadline: 1731575751575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:51,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:51,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:51,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:51,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:51,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:51,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:51,611 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/0835a71643f944cea5b3c875ea880c16 2024-11-14T09:14:51,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/2df63f3e815143f5acd4e65b88ca88ec as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/2df63f3e815143f5acd4e65b88ca88ec 2024-11-14T09:14:51,620 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/2df63f3e815143f5acd4e65b88ca88ec, entries=250, sequenceid=95, filesize=16.3 K 2024-11-14T09:14:51,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/b1f84d82d7984ea7ba859b76c1504abc as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/b1f84d82d7984ea7ba859b76c1504abc 2024-11-14T09:14:51,624 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/b1f84d82d7984ea7ba859b76c1504abc, entries=150, sequenceid=95, filesize=11.7 K 2024-11-14T09:14:51,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/0835a71643f944cea5b3c875ea880c16 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/0835a71643f944cea5b3c875ea880c16 2024-11-14T09:14:51,628 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/0835a71643f944cea5b3c875ea880c16, entries=150, sequenceid=95, filesize=11.7 K 2024-11-14T09:14:51,629 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 8664731a05c52a198640968a7aa48527 in 1274ms, sequenceid=95, compaction requested=true 2024-11-14T09:14:51,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8664731a05c52a198640968a7aa48527: 2024-11-14T09:14:51,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8664731a05c52a198640968a7aa48527:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:14:51,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:51,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8664731a05c52a198640968a7aa48527:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:14:51,629 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:14:51,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:51,629 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:14:51,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8664731a05c52a198640968a7aa48527:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:14:51,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:14:51,630 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52787 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:14:51,630 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:14:51,630 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): 8664731a05c52a198640968a7aa48527/A is initiating minor compaction (all files) 2024-11-14T09:14:51,631 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): 8664731a05c52a198640968a7aa48527/B is initiating minor compaction (all files) 2024-11-14T09:14:51,631 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8664731a05c52a198640968a7aa48527/A in TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:51,631 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8664731a05c52a198640968a7aa48527/B in TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:51,631 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/efb9fa9402e741588dc5f0a8cacd4c1d, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/aa359d9fcbf54b9ea0aa546cf88c653e, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/4892896493724401aa747e870dfe2be2, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/2df63f3e815143f5acd4e65b88ca88ec] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp, totalSize=51.5 K 2024-11-14T09:14:51,631 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/4daad9fc31564870ba30f63a88de12a5, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/eb8ae91782014f808a9d4c8efac1e7cb, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/af396144976f420d857994aa08f861a5, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/b1f84d82d7984ea7ba859b76c1504abc] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp, totalSize=47.0 K 2024-11-14T09:14:51,631 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 4daad9fc31564870ba30f63a88de12a5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731575687385 2024-11-14T09:14:51,631 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting efb9fa9402e741588dc5f0a8cacd4c1d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731575687385 2024-11-14T09:14:51,632 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa359d9fcbf54b9ea0aa546cf88c653e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1731575688543 2024-11-14T09:14:51,632 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting eb8ae91782014f808a9d4c8efac1e7cb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1731575688543 2024-11-14T09:14:51,633 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4892896493724401aa747e870dfe2be2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731575689187 2024-11-14T09:14:51,633 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting af396144976f420d857994aa08f861a5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731575689187 2024-11-14T09:14:51,633 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting b1f84d82d7984ea7ba859b76c1504abc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731575690349 2024-11-14T09:14:51,633 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2df63f3e815143f5acd4e65b88ca88ec, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731575690349 2024-11-14T09:14:51,643 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8664731a05c52a198640968a7aa48527#B#compaction#226 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:14:51,643 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8664731a05c52a198640968a7aa48527#A#compaction#225 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:14:51,643 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/f9d5f3ced1874fb38095f90db028a8dc is 50, key is test_row_0/B:col10/1731575690354/Put/seqid=0 2024-11-14T09:14:51,643 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/c62875c5ccc14c9a8d6b781c248922b8 is 50, key is test_row_0/A:col10/1731575690354/Put/seqid=0 2024-11-14T09:14:51,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742097_1273 (size=12241) 2024-11-14T09:14:51,661 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/c62875c5ccc14c9a8d6b781c248922b8 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/c62875c5ccc14c9a8d6b781c248922b8 2024-11-14T09:14:51,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742098_1274 (size=12241) 2024-11-14T09:14:51,668 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8664731a05c52a198640968a7aa48527/A of 8664731a05c52a198640968a7aa48527 into c62875c5ccc14c9a8d6b781c248922b8(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:14:51,668 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8664731a05c52a198640968a7aa48527: 2024-11-14T09:14:51,668 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., storeName=8664731a05c52a198640968a7aa48527/A, priority=12, startTime=1731575691629; duration=0sec 2024-11-14T09:14:51,668 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:14:51,668 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8664731a05c52a198640968a7aa48527:A 2024-11-14T09:14:51,668 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:14:51,670 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:14:51,670 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): 8664731a05c52a198640968a7aa48527/C is initiating minor compaction (all files) 2024-11-14T09:14:51,671 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8664731a05c52a198640968a7aa48527/C in TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:51,671 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/c6d0aac053af4eabbab58711b99753e6, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/2cff30a1149e4424b52e3b50c527fa94, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/a63b0c8362ea424da1edae51e3efccf7, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/0835a71643f944cea5b3c875ea880c16] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp, totalSize=47.0 K 2024-11-14T09:14:51,672 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6d0aac053af4eabbab58711b99753e6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731575687385 2024-11-14T09:14:51,674 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2cff30a1149e4424b52e3b50c527fa94, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1731575688543 2024-11-14T09:14:51,674 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting a63b0c8362ea424da1edae51e3efccf7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731575689187 2024-11-14T09:14:51,675 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0835a71643f944cea5b3c875ea880c16, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731575690349 2024-11-14T09:14:51,675 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/f9d5f3ced1874fb38095f90db028a8dc as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/f9d5f3ced1874fb38095f90db028a8dc 2024-11-14T09:14:51,684 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8664731a05c52a198640968a7aa48527/B of 8664731a05c52a198640968a7aa48527 into f9d5f3ced1874fb38095f90db028a8dc(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:14:51,684 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8664731a05c52a198640968a7aa48527: 2024-11-14T09:14:51,684 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., storeName=8664731a05c52a198640968a7aa48527/B, priority=12, startTime=1731575691629; duration=0sec 2024-11-14T09:14:51,684 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:51,684 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8664731a05c52a198640968a7aa48527:B 2024-11-14T09:14:51,690 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8664731a05c52a198640968a7aa48527#C#compaction#227 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:14:51,690 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/9aa697c613e346fbb15607fca0f20744 is 50, key is test_row_0/C:col10/1731575690354/Put/seqid=0 2024-11-14T09:14:51,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742099_1275 (size=12241) 2024-11-14T09:14:51,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-14T09:14:51,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:51,717 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 8664731a05c52a198640968a7aa48527 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-14T09:14:51,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=A 2024-11-14T09:14:51,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:14:51,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=B 2024-11-14T09:14:51,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:14:51,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=C 2024-11-14T09:14:51,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:14:51,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/2442a1b87d4446ef846301da5a9e8394 is 50, key is test_row_0/A:col10/1731575690404/Put/seqid=0 2024-11-14T09:14:51,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742100_1276 (size=12051) 2024-11-14T09:14:51,743 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/2442a1b87d4446ef846301da5a9e8394 2024-11-14T09:14:51,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/8b0904923b094cb2b03d6c8ecf9a1edc is 50, key is test_row_0/B:col10/1731575690404/Put/seqid=0 2024-11-14T09:14:51,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742101_1277 (size=12051) 2024-11-14T09:14:51,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-14T09:14:52,104 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/9aa697c613e346fbb15607fca0f20744 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/9aa697c613e346fbb15607fca0f20744 2024-11-14T09:14:52,109 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8664731a05c52a198640968a7aa48527/C of 8664731a05c52a198640968a7aa48527 into 9aa697c613e346fbb15607fca0f20744(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:14:52,109 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8664731a05c52a198640968a7aa48527: 2024-11-14T09:14:52,109 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., storeName=8664731a05c52a198640968a7aa48527/C, priority=12, startTime=1731575691629; duration=0sec 2024-11-14T09:14:52,110 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:52,110 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8664731a05c52a198640968a7aa48527:C 2024-11-14T09:14:52,159 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/8b0904923b094cb2b03d6c8ecf9a1edc 2024-11-14T09:14:52,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/12135aabc6c44912a7a7b3ef8c08f1c5 is 50, key is test_row_0/C:col10/1731575690404/Put/seqid=0 2024-11-14T09:14:52,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742102_1278 (size=12051) 2024-11-14T09:14:52,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-14T09:14:52,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on 8664731a05c52a198640968a7aa48527 2024-11-14T09:14:52,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. as already flushing 2024-11-14T09:14:52,573 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/12135aabc6c44912a7a7b3ef8c08f1c5 2024-11-14T09:14:52,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/2442a1b87d4446ef846301da5a9e8394 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/2442a1b87d4446ef846301da5a9e8394 2024-11-14T09:14:52,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:52,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731575752574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:52,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:52,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58142 deadline: 1731575752575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:52,580 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_2' in 8664731a05c52a198640968a7aa48527 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:14:52,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:52,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:52,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:52,583 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:52,583 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:52,583 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:52,584 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/2442a1b87d4446ef846301da5a9e8394, entries=150, sequenceid=132, filesize=11.8 K 2024-11-14T09:14:52,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/8b0904923b094cb2b03d6c8ecf9a1edc as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/8b0904923b094cb2b03d6c8ecf9a1edc 2024-11-14T09:14:52,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:52,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58184 deadline: 1731575752585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:52,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:52,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58168 deadline: 1731575752585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:52,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:52,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58122 deadline: 1731575752586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:52,588 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_2' in 8664731a05c52a198640968a7aa48527 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:14:52,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:52,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:52,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:52,589 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_2' in 8664731a05c52a198640968a7aa48527 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:14:52,589 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_2' in 8664731a05c52a198640968a7aa48527 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:14:52,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:52,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:52,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:52,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:52,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:52,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:52,591 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/8b0904923b094cb2b03d6c8ecf9a1edc, entries=150, sequenceid=132, filesize=11.8 K 2024-11-14T09:14:52,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/12135aabc6c44912a7a7b3ef8c08f1c5 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/12135aabc6c44912a7a7b3ef8c08f1c5 2024-11-14T09:14:52,601 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/12135aabc6c44912a7a7b3ef8c08f1c5, entries=150, sequenceid=132, filesize=11.8 K 2024-11-14T09:14:52,601 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 8664731a05c52a198640968a7aa48527 in 884ms, sequenceid=132, compaction requested=false 2024-11-14T09:14:52,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 8664731a05c52a198640968a7aa48527: 2024-11-14T09:14:52,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:52,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-14T09:14:52,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-14T09:14:52,607 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-14T09:14:52,607 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3450 sec 2024-11-14T09:14:52,609 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.3520 sec 2024-11-14T09:14:52,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on 8664731a05c52a198640968a7aa48527 2024-11-14T09:14:52,698 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8664731a05c52a198640968a7aa48527 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-14T09:14:52,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=A 2024-11-14T09:14:52,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:52,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=B 2024-11-14T09:14:52,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:52,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=C 2024-11-14T09:14:52,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:52,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/5afc0de76abf4af187352ddb72418307 is 50, key is test_row_0/A:col10/1731575692573/Put/seqid=0 2024-11-14T09:14:52,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742103_1279 (size=14541) 2024-11-14T09:14:52,716 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/5afc0de76abf4af187352ddb72418307 2024-11-14T09:14:52,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/bd70e73d6438497d872fc2ffeed2f44c is 50, key is test_row_0/B:col10/1731575692573/Put/seqid=0 2024-11-14T09:14:52,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742104_1280 (size=12151) 2024-11-14T09:14:52,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:52,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731575752782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:52,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:52,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:52,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:52,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:52,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731575752895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:52,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:52,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:52,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:53,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:53,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731575753104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:53,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:53,106 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:53,106 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:53,134 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/bd70e73d6438497d872fc2ffeed2f44c 2024-11-14T09:14:53,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/c8017cc3c6c5448c8c5b7cce91811bcd is 50, key is test_row_0/C:col10/1731575692573/Put/seqid=0 2024-11-14T09:14:53,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742105_1281 (size=12151) 2024-11-14T09:14:53,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-14T09:14:53,405 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:14:53,405 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:14:53,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-14T09:14:53,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T09:14:53,408 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:14:53,409 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:14:53,409 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:14:53,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:53,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731575753415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:53,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:53,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:53,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:53,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T09:14:53,546 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/c8017cc3c6c5448c8c5b7cce91811bcd 2024-11-14T09:14:53,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/5afc0de76abf4af187352ddb72418307 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/5afc0de76abf4af187352ddb72418307 2024-11-14T09:14:53,554 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/5afc0de76abf4af187352ddb72418307, entries=200, sequenceid=145, filesize=14.2 K 2024-11-14T09:14:53,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/bd70e73d6438497d872fc2ffeed2f44c as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/bd70e73d6438497d872fc2ffeed2f44c 2024-11-14T09:14:53,559 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/bd70e73d6438497d872fc2ffeed2f44c, entries=150, sequenceid=145, filesize=11.9 K 2024-11-14T09:14:53,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/c8017cc3c6c5448c8c5b7cce91811bcd as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/c8017cc3c6c5448c8c5b7cce91811bcd 2024-11-14T09:14:53,562 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-14T09:14:53,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:53,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. as already flushing 2024-11-14T09:14:53,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:53,562 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:53,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:53,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:53,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/c8017cc3c6c5448c8c5b7cce91811bcd, entries=150, sequenceid=145, filesize=11.9 K 2024-11-14T09:14:53,566 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 8664731a05c52a198640968a7aa48527 in 868ms, sequenceid=145, compaction requested=true 2024-11-14T09:14:53,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8664731a05c52a198640968a7aa48527: 2024-11-14T09:14:53,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8664731a05c52a198640968a7aa48527:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:14:53,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:53,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8664731a05c52a198640968a7aa48527:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:14:53,566 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:14:53,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:53,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8664731a05c52a198640968a7aa48527:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:14:53,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:14:53,566 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:14:53,567 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38833 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:14:53,567 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): 8664731a05c52a198640968a7aa48527/A is initiating minor compaction (all files) 2024-11-14T09:14:53,567 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36443 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:14:53,568 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8664731a05c52a198640968a7aa48527/A in TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:53,568 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): 8664731a05c52a198640968a7aa48527/B is initiating minor compaction (all files) 2024-11-14T09:14:53,568 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8664731a05c52a198640968a7aa48527/B in TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:53,568 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/c62875c5ccc14c9a8d6b781c248922b8, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/2442a1b87d4446ef846301da5a9e8394, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/5afc0de76abf4af187352ddb72418307] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp, totalSize=37.9 K 2024-11-14T09:14:53,568 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/f9d5f3ced1874fb38095f90db028a8dc, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/8b0904923b094cb2b03d6c8ecf9a1edc, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/bd70e73d6438497d872fc2ffeed2f44c] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp, totalSize=35.6 K 2024-11-14T09:14:53,569 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting c62875c5ccc14c9a8d6b781c248922b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731575690349 2024-11-14T09:14:53,569 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting f9d5f3ced1874fb38095f90db028a8dc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731575690349 2024-11-14T09:14:53,570 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b0904923b094cb2b03d6c8ecf9a1edc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731575690404 2024-11-14T09:14:53,570 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2442a1b87d4446ef846301da5a9e8394, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731575690404 2024-11-14T09:14:53,571 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting bd70e73d6438497d872fc2ffeed2f44c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731575692545 2024-11-14T09:14:53,571 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5afc0de76abf4af187352ddb72418307, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731575692545 2024-11-14T09:14:53,600 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8664731a05c52a198640968a7aa48527#B#compaction#234 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:14:53,600 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/a833de38560a4db28dd567434c3dee08 is 50, key is test_row_0/B:col10/1731575692573/Put/seqid=0 2024-11-14T09:14:53,604 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8664731a05c52a198640968a7aa48527#A#compaction#235 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:14:53,605 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/074f96b812d046c792a28eabc64d317f is 50, key is test_row_0/A:col10/1731575692573/Put/seqid=0 2024-11-14T09:14:53,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742106_1282 (size=12493) 2024-11-14T09:14:53,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742107_1283 (size=12493) 2024-11-14T09:14:53,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-14T09:14:53,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:53,715 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 8664731a05c52a198640968a7aa48527 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-14T09:14:53,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=A 2024-11-14T09:14:53,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:53,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=B 2024-11-14T09:14:53,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:53,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=C 2024-11-14T09:14:53,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:53,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/ddc04f5f4c5e4b509df609f3d3e1b6d2 is 50, key is test_row_0/A:col10/1731575692781/Put/seqid=0 2024-11-14T09:14:53,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T09:14:53,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742108_1284 (size=12151) 2024-11-14T09:14:53,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on 8664731a05c52a198640968a7aa48527 2024-11-14T09:14:53,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. as already flushing 2024-11-14T09:14:53,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:53,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731575753963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:53,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:53,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:53,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:54,032 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/a833de38560a4db28dd567434c3dee08 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/a833de38560a4db28dd567434c3dee08 2024-11-14T09:14:54,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T09:14:54,038 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8664731a05c52a198640968a7aa48527/B of 8664731a05c52a198640968a7aa48527 into a833de38560a4db28dd567434c3dee08(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:14:54,038 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8664731a05c52a198640968a7aa48527: 2024-11-14T09:14:54,038 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., storeName=8664731a05c52a198640968a7aa48527/B, priority=13, startTime=1731575693566; duration=0sec 2024-11-14T09:14:54,038 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:14:54,038 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8664731a05c52a198640968a7aa48527:B 2024-11-14T09:14:54,038 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:14:54,039 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/074f96b812d046c792a28eabc64d317f as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/074f96b812d046c792a28eabc64d317f 2024-11-14T09:14:54,040 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36443 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:14:54,041 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): 8664731a05c52a198640968a7aa48527/C is initiating minor compaction (all files) 2024-11-14T09:14:54,041 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8664731a05c52a198640968a7aa48527/C in TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:54,041 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/9aa697c613e346fbb15607fca0f20744, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/12135aabc6c44912a7a7b3ef8c08f1c5, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/c8017cc3c6c5448c8c5b7cce91811bcd] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp, totalSize=35.6 K 2024-11-14T09:14:54,041 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 9aa697c613e346fbb15607fca0f20744, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731575690349 2024-11-14T09:14:54,042 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 12135aabc6c44912a7a7b3ef8c08f1c5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731575690404 2024-11-14T09:14:54,042 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting c8017cc3c6c5448c8c5b7cce91811bcd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731575692545 2024-11-14T09:14:54,044 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8664731a05c52a198640968a7aa48527/A of 8664731a05c52a198640968a7aa48527 into 074f96b812d046c792a28eabc64d317f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:14:54,044 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8664731a05c52a198640968a7aa48527: 2024-11-14T09:14:54,044 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., storeName=8664731a05c52a198640968a7aa48527/A, priority=13, startTime=1731575693566; duration=0sec 2024-11-14T09:14:54,044 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:54,044 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8664731a05c52a198640968a7aa48527:A 2024-11-14T09:14:54,051 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8664731a05c52a198640968a7aa48527#C#compaction#237 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:14:54,052 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/2c4a181e23be4766872696eac98a8839 is 50, key is test_row_0/C:col10/1731575692573/Put/seqid=0 2024-11-14T09:14:54,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742109_1285 (size=12493) 2024-11-14T09:14:54,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:54,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731575754075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:54,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:54,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:54,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:54,130 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/ddc04f5f4c5e4b509df609f3d3e1b6d2 2024-11-14T09:14:54,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/4263f94563a04cc991f0a4677b890e20 is 50, key is test_row_0/B:col10/1731575692781/Put/seqid=0 2024-11-14T09:14:54,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742110_1286 (size=12151) 2024-11-14T09:14:54,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:54,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731575754285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:54,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:54,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:54,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:54,461 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/2c4a181e23be4766872696eac98a8839 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/2c4a181e23be4766872696eac98a8839 2024-11-14T09:14:54,466 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8664731a05c52a198640968a7aa48527/C of 8664731a05c52a198640968a7aa48527 into 2c4a181e23be4766872696eac98a8839(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:14:54,466 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8664731a05c52a198640968a7aa48527: 2024-11-14T09:14:54,466 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., storeName=8664731a05c52a198640968a7aa48527/C, priority=13, startTime=1731575693566; duration=0sec 2024-11-14T09:14:54,466 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:54,466 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8664731a05c52a198640968a7aa48527:C 2024-11-14T09:14:54,542 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/4263f94563a04cc991f0a4677b890e20 2024-11-14T09:14:54,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T09:14:54,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/05ed99bbd20242edade72be64e7d6da7 is 50, key is test_row_0/C:col10/1731575692781/Put/seqid=0 2024-11-14T09:14:54,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742111_1287 (size=12151) 2024-11-14T09:14:54,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:54,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58142 deadline: 1731575754595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:54,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:54,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731575754595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:54,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:54,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:54,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:54,598 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_2' in 8664731a05c52a198640968a7aa48527 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:14:54,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:54,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:54,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:54,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:54,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58184 deadline: 1731575754604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:54,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:54,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:54,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58122 deadline: 1731575754605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:54,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58168 deadline: 1731575754605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:54,607 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_2' in 8664731a05c52a198640968a7aa48527 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:14:54,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:54,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:54,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:54,608 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_2' in 8664731a05c52a198640968a7aa48527 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:14:54,608 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_2' in 8664731a05c52a198640968a7aa48527 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:14:54,609 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:54,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:54,609 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:54,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:54,609 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:54,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:54,959 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/05ed99bbd20242edade72be64e7d6da7 2024-11-14T09:14:54,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/ddc04f5f4c5e4b509df609f3d3e1b6d2 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/ddc04f5f4c5e4b509df609f3d3e1b6d2 2024-11-14T09:14:54,970 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/ddc04f5f4c5e4b509df609f3d3e1b6d2, entries=150, sequenceid=165, filesize=11.9 K 2024-11-14T09:14:54,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/4263f94563a04cc991f0a4677b890e20 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/4263f94563a04cc991f0a4677b890e20 2024-11-14T09:14:54,975 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/4263f94563a04cc991f0a4677b890e20, entries=150, sequenceid=165, filesize=11.9 K 2024-11-14T09:14:54,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/05ed99bbd20242edade72be64e7d6da7 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/05ed99bbd20242edade72be64e7d6da7 2024-11-14T09:14:54,979 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/05ed99bbd20242edade72be64e7d6da7, entries=150, sequenceid=165, filesize=11.9 K 2024-11-14T09:14:54,980 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 8664731a05c52a198640968a7aa48527 in 1265ms, sequenceid=165, compaction requested=false 2024-11-14T09:14:54,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 8664731a05c52a198640968a7aa48527: 2024-11-14T09:14:54,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:54,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-14T09:14:54,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-14T09:14:54,983 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-14T09:14:54,983 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5730 sec 2024-11-14T09:14:54,985 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.5780 sec 2024-11-14T09:14:55,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on 8664731a05c52a198640968a7aa48527 2024-11-14T09:14:55,106 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8664731a05c52a198640968a7aa48527 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-14T09:14:55,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=A 2024-11-14T09:14:55,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:55,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=B 2024-11-14T09:14:55,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:55,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=C 2024-11-14T09:14:55,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:55,112 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/95e26fc09aa34f31af344e6f909b47ec is 50, key is test_row_0/A:col10/1731575693957/Put/seqid=0 2024-11-14T09:14:55,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742112_1288 (size=14541) 2024-11-14T09:14:55,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:55,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731575755180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:55,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:55,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:55,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:55,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:55,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731575755295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:55,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:55,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:55,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:55,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:55,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731575755505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:55,508 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:55,508 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:55,508 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:55,517 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/95e26fc09aa34f31af344e6f909b47ec 2024-11-14T09:14:55,524 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/5f702a1007c24349af6f5971ba2e2cc0 is 50, key is test_row_0/B:col10/1731575693957/Put/seqid=0 2024-11-14T09:14:55,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742113_1289 (size=12151) 2024-11-14T09:14:55,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T09:14:55,555 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:14:55,556 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:14:55,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-14T09:14:55,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-14T09:14:55,558 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:14:55,558 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:14:55,559 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:14:55,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-14T09:14:55,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T09:14:55,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:55,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. as already flushing 2024-11-14T09:14:55,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:55,712 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:55,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:55,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:55,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:55,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731575755814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:55,818 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:55,818 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:55,818 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:55,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T09:14:55,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:55,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. as already flushing 2024-11-14T09:14:55,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:55,865 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:55,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:55,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:55,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-14T09:14:55,929 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/5f702a1007c24349af6f5971ba2e2cc0 2024-11-14T09:14:55,937 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/ce32dbd84beb49c2abb7b3e66b6f5e06 is 50, key is test_row_0/C:col10/1731575693957/Put/seqid=0 2024-11-14T09:14:55,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742114_1290 (size=12151) 2024-11-14T09:14:56,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T09:14:56,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:56,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. as already flushing 2024-11-14T09:14:56,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:56,018 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:56,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:56,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:56,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T09:14:56,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:56,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. as already flushing 2024-11-14T09:14:56,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:56,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:56,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:56,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:56,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-14T09:14:56,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T09:14:56,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:56,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. as already flushing 2024-11-14T09:14:56,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:56,325 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:56,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:56,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:56,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:56,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731575756325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:56,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:56,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:56,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:56,341 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/ce32dbd84beb49c2abb7b3e66b6f5e06 2024-11-14T09:14:56,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/95e26fc09aa34f31af344e6f909b47ec as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/95e26fc09aa34f31af344e6f909b47ec 2024-11-14T09:14:56,349 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/95e26fc09aa34f31af344e6f909b47ec, entries=200, sequenceid=181, filesize=14.2 K 2024-11-14T09:14:56,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/5f702a1007c24349af6f5971ba2e2cc0 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/5f702a1007c24349af6f5971ba2e2cc0 2024-11-14T09:14:56,354 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/5f702a1007c24349af6f5971ba2e2cc0, entries=150, sequenceid=181, filesize=11.9 K 2024-11-14T09:14:56,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/ce32dbd84beb49c2abb7b3e66b6f5e06 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/ce32dbd84beb49c2abb7b3e66b6f5e06 2024-11-14T09:14:56,359 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/ce32dbd84beb49c2abb7b3e66b6f5e06, entries=150, sequenceid=181, filesize=11.9 K 2024-11-14T09:14:56,360 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 8664731a05c52a198640968a7aa48527 in 1254ms, sequenceid=181, compaction requested=true 2024-11-14T09:14:56,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8664731a05c52a198640968a7aa48527: 2024-11-14T09:14:56,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8664731a05c52a198640968a7aa48527:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:14:56,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:56,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8664731a05c52a198640968a7aa48527:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:14:56,360 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:14:56,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:56,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8664731a05c52a198640968a7aa48527:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:14:56,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:14:56,360 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:14:56,361 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:14:56,361 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:14:56,361 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): 8664731a05c52a198640968a7aa48527/B is initiating minor compaction (all files) 2024-11-14T09:14:56,361 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): 8664731a05c52a198640968a7aa48527/A is initiating minor compaction (all files) 2024-11-14T09:14:56,361 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8664731a05c52a198640968a7aa48527/B in TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:56,361 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8664731a05c52a198640968a7aa48527/A in TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:56,361 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/a833de38560a4db28dd567434c3dee08, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/4263f94563a04cc991f0a4677b890e20, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/5f702a1007c24349af6f5971ba2e2cc0] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp, totalSize=35.9 K 2024-11-14T09:14:56,361 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/074f96b812d046c792a28eabc64d317f, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/ddc04f5f4c5e4b509df609f3d3e1b6d2, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/95e26fc09aa34f31af344e6f909b47ec] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp, totalSize=38.3 K 2024-11-14T09:14:56,362 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 074f96b812d046c792a28eabc64d317f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731575692545 2024-11-14T09:14:56,362 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting a833de38560a4db28dd567434c3dee08, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731575692545 2024-11-14T09:14:56,362 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 4263f94563a04cc991f0a4677b890e20, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731575692766 2024-11-14T09:14:56,362 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting ddc04f5f4c5e4b509df609f3d3e1b6d2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731575692766 2024-11-14T09:14:56,362 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f702a1007c24349af6f5971ba2e2cc0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731575693926 2024-11-14T09:14:56,362 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95e26fc09aa34f31af344e6f909b47ec, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731575693926 2024-11-14T09:14:56,374 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8664731a05c52a198640968a7aa48527#B#compaction#244 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:14:56,374 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8664731a05c52a198640968a7aa48527#A#compaction#243 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:14:56,374 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/72a6189ff6c74bb1bb1df1a6fe707178 is 50, key is test_row_0/B:col10/1731575693957/Put/seqid=0 2024-11-14T09:14:56,375 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/1e73dbfc059b4b869b7a48c7ed25632e is 50, key is test_row_0/A:col10/1731575693957/Put/seqid=0 2024-11-14T09:14:56,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742115_1291 (size=12595) 2024-11-14T09:14:56,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742116_1292 (size=12595) 2024-11-14T09:14:56,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T09:14:56,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:56,479 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 8664731a05c52a198640968a7aa48527 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-14T09:14:56,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=A 2024-11-14T09:14:56,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:56,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=B 2024-11-14T09:14:56,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:56,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=C 2024-11-14T09:14:56,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:56,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/df844f87055140409c43490383c5676c is 50, key is test_row_0/A:col10/1731575695168/Put/seqid=0 2024-11-14T09:14:56,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742117_1293 (size=12151) 2024-11-14T09:14:56,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-14T09:14:56,786 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/72a6189ff6c74bb1bb1df1a6fe707178 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/72a6189ff6c74bb1bb1df1a6fe707178 2024-11-14T09:14:56,793 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8664731a05c52a198640968a7aa48527/B of 8664731a05c52a198640968a7aa48527 into 72a6189ff6c74bb1bb1df1a6fe707178(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:14:56,793 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/1e73dbfc059b4b869b7a48c7ed25632e as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/1e73dbfc059b4b869b7a48c7ed25632e 2024-11-14T09:14:56,793 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8664731a05c52a198640968a7aa48527: 2024-11-14T09:14:56,793 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., storeName=8664731a05c52a198640968a7aa48527/B, priority=13, startTime=1731575696360; duration=0sec 2024-11-14T09:14:56,794 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:14:56,794 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8664731a05c52a198640968a7aa48527:B 2024-11-14T09:14:56,794 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:14:56,795 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:14:56,795 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): 8664731a05c52a198640968a7aa48527/C is initiating minor compaction (all files) 2024-11-14T09:14:56,795 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8664731a05c52a198640968a7aa48527/C in TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:56,796 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/2c4a181e23be4766872696eac98a8839, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/05ed99bbd20242edade72be64e7d6da7, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/ce32dbd84beb49c2abb7b3e66b6f5e06] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp, totalSize=35.9 K 2024-11-14T09:14:56,796 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c4a181e23be4766872696eac98a8839, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731575692545 2024-11-14T09:14:56,797 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 05ed99bbd20242edade72be64e7d6da7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731575692766 2024-11-14T09:14:56,800 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8664731a05c52a198640968a7aa48527/A of 8664731a05c52a198640968a7aa48527 into 1e73dbfc059b4b869b7a48c7ed25632e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:14:56,800 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8664731a05c52a198640968a7aa48527: 2024-11-14T09:14:56,800 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., storeName=8664731a05c52a198640968a7aa48527/A, priority=13, startTime=1731575696360; duration=0sec 2024-11-14T09:14:56,800 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:56,800 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8664731a05c52a198640968a7aa48527:A 2024-11-14T09:14:56,802 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting ce32dbd84beb49c2abb7b3e66b6f5e06, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731575693926 2024-11-14T09:14:56,810 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8664731a05c52a198640968a7aa48527#C#compaction#246 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:14:56,811 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/e245f91696334f6fac42c55910e4fdcb is 50, key is test_row_0/C:col10/1731575693957/Put/seqid=0 2024-11-14T09:14:56,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742118_1294 (size=12595) 2024-11-14T09:14:56,889 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/df844f87055140409c43490383c5676c 2024-11-14T09:14:56,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/a059c52f391d4c2b89c756d040f1bf46 is 50, key is test_row_0/B:col10/1731575695168/Put/seqid=0 2024-11-14T09:14:56,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742119_1295 (size=12151) 2024-11-14T09:14:56,903 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/a059c52f391d4c2b89c756d040f1bf46 2024-11-14T09:14:56,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/dc22d4eaa5114ef1986204aaaf7df1ea is 50, key is test_row_0/C:col10/1731575695168/Put/seqid=0 2024-11-14T09:14:56,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742120_1296 (size=12151) 2024-11-14T09:14:56,918 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/dc22d4eaa5114ef1986204aaaf7df1ea 2024-11-14T09:14:56,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/df844f87055140409c43490383c5676c as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/df844f87055140409c43490383c5676c 2024-11-14T09:14:56,928 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/df844f87055140409c43490383c5676c, entries=150, sequenceid=200, filesize=11.9 K 2024-11-14T09:14:56,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/a059c52f391d4c2b89c756d040f1bf46 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/a059c52f391d4c2b89c756d040f1bf46 2024-11-14T09:14:56,935 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/a059c52f391d4c2b89c756d040f1bf46, entries=150, sequenceid=200, filesize=11.9 K 2024-11-14T09:14:56,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/dc22d4eaa5114ef1986204aaaf7df1ea as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/dc22d4eaa5114ef1986204aaaf7df1ea 2024-11-14T09:14:56,941 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/dc22d4eaa5114ef1986204aaaf7df1ea, entries=150, sequenceid=200, filesize=11.9 K 2024-11-14T09:14:56,942 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=0 B/0 for 8664731a05c52a198640968a7aa48527 in 464ms, sequenceid=200, compaction requested=false 2024-11-14T09:14:56,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 8664731a05c52a198640968a7aa48527: 2024-11-14T09:14:56,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:56,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-14T09:14:56,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-14T09:14:56,947 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-14T09:14:56,947 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3850 sec 2024-11-14T09:14:56,951 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.3930 sec 2024-11-14T09:14:57,234 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/e245f91696334f6fac42c55910e4fdcb as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/e245f91696334f6fac42c55910e4fdcb 2024-11-14T09:14:57,239 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8664731a05c52a198640968a7aa48527/C of 8664731a05c52a198640968a7aa48527 into e245f91696334f6fac42c55910e4fdcb(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:14:57,239 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8664731a05c52a198640968a7aa48527: 2024-11-14T09:14:57,239 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., storeName=8664731a05c52a198640968a7aa48527/C, priority=13, startTime=1731575696360; duration=0sec 2024-11-14T09:14:57,239 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:57,239 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8664731a05c52a198640968a7aa48527:C 2024-11-14T09:14:57,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on 8664731a05c52a198640968a7aa48527 2024-11-14T09:14:57,349 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8664731a05c52a198640968a7aa48527 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T09:14:57,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=A 2024-11-14T09:14:57,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:57,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=B 2024-11-14T09:14:57,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:57,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=C 2024-11-14T09:14:57,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:14:57,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/4e6de91173f0494da9c9be9b5fea1678 is 50, key is test_row_0/A:col10/1731575697348/Put/seqid=0 2024-11-14T09:14:57,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742121_1297 (size=12151) 2024-11-14T09:14:57,432 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731575697367, max timestamp=1731575697428Num uniques 859; 2024-11-14T09:14:57,432 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:14:57,432 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731575697367, max timestamp=1731575697428Num uniques 859; 2024-11-14T09:14:57,432 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:14:57,432 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731575697367, max timestamp=1731575697428Num uniques 859; 2024-11-14T09:14:57,432 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:14:57,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:57,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731575757523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:57,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:57,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:57,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:57,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:57,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731575757635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:57,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:57,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:57,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:57,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-14T09:14:57,705 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:14:57,705 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:14:57,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-14T09:14:57,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T09:14:57,707 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:14:57,708 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:14:57,708 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:14:57,763 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/4e6de91173f0494da9c9be9b5fea1678 2024-11-14T09:14:57,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/95afab003c5e46b8949968d115ef2052 is 50, key is test_row_0/B:col10/1731575697348/Put/seqid=0 2024-11-14T09:14:57,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742122_1298 (size=12151) 2024-11-14T09:14:57,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T09:14:57,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:57,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731575757845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:57,848 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:57,848 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:57,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:57,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-14T09:14:57,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:57,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. as already flushing 2024-11-14T09:14:57,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:57,860 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:57,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:57,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:58,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-14T09:14:58,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:58,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. as already flushing 2024-11-14T09:14:58,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:58,014 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:58,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:58,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:58,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T09:14:58,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:58,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731575758154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:58,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:58,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:58,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:58,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-14T09:14:58,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:58,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. as already flushing 2024-11-14T09:14:58,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:58,167 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:58,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:58,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:58,175 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/95afab003c5e46b8949968d115ef2052 2024-11-14T09:14:58,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/26534f0444824557b8192887022d3274 is 50, key is test_row_0/C:col10/1731575697348/Put/seqid=0 2024-11-14T09:14:58,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742123_1299 (size=12151) 2024-11-14T09:14:58,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-14T09:14:58,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:58,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. as already flushing 2024-11-14T09:14:58,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:58,320 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:58,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:58,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:58,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T09:14:58,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-14T09:14:58,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:58,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. as already flushing 2024-11-14T09:14:58,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:58,473 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:58,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:58,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:14:58,591 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/26534f0444824557b8192887022d3274 2024-11-14T09:14:58,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/4e6de91173f0494da9c9be9b5fea1678 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/4e6de91173f0494da9c9be9b5fea1678 2024-11-14T09:14:58,605 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/4e6de91173f0494da9c9be9b5fea1678, entries=150, sequenceid=213, filesize=11.9 K 2024-11-14T09:14:58,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/95afab003c5e46b8949968d115ef2052 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/95afab003c5e46b8949968d115ef2052 2024-11-14T09:14:58,609 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/95afab003c5e46b8949968d115ef2052, entries=150, sequenceid=213, filesize=11.9 K 2024-11-14T09:14:58,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/26534f0444824557b8192887022d3274 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/26534f0444824557b8192887022d3274 2024-11-14T09:14:58,614 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/26534f0444824557b8192887022d3274, entries=150, sequenceid=213, filesize=11.9 K 2024-11-14T09:14:58,614 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 8664731a05c52a198640968a7aa48527 in 1265ms, sequenceid=213, compaction requested=true 2024-11-14T09:14:58,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8664731a05c52a198640968a7aa48527: 2024-11-14T09:14:58,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8664731a05c52a198640968a7aa48527:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:14:58,615 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:14:58,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:58,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8664731a05c52a198640968a7aa48527:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:14:58,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:58,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8664731a05c52a198640968a7aa48527:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:14:58,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:14:58,615 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:14:58,616 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:14:58,616 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:14:58,616 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): 8664731a05c52a198640968a7aa48527/A is initiating minor compaction (all files) 2024-11-14T09:14:58,616 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): 8664731a05c52a198640968a7aa48527/B is initiating minor compaction (all files) 2024-11-14T09:14:58,616 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8664731a05c52a198640968a7aa48527/A in TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:58,616 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8664731a05c52a198640968a7aa48527/B in TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:58,616 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/1e73dbfc059b4b869b7a48c7ed25632e, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/df844f87055140409c43490383c5676c, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/4e6de91173f0494da9c9be9b5fea1678] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp, totalSize=36.0 K 2024-11-14T09:14:58,616 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/72a6189ff6c74bb1bb1df1a6fe707178, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/a059c52f391d4c2b89c756d040f1bf46, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/95afab003c5e46b8949968d115ef2052] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp, totalSize=36.0 K 2024-11-14T09:14:58,616 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e73dbfc059b4b869b7a48c7ed25632e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731575693926 2024-11-14T09:14:58,616 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 72a6189ff6c74bb1bb1df1a6fe707178, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731575693926 2024-11-14T09:14:58,617 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting df844f87055140409c43490383c5676c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731575695168 2024-11-14T09:14:58,617 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting a059c52f391d4c2b89c756d040f1bf46, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731575695168 2024-11-14T09:14:58,617 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e6de91173f0494da9c9be9b5fea1678, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731575697341 2024-11-14T09:14:58,617 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 95afab003c5e46b8949968d115ef2052, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731575697341 2024-11-14T09:14:58,625 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8664731a05c52a198640968a7aa48527#B#compaction#252 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:14:58,626 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/b8fb581b68d54d6789ed86067acb29af is 50, key is test_row_0/B:col10/1731575697348/Put/seqid=0 2024-11-14T09:14:58,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-14T09:14:58,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:58,626 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 8664731a05c52a198640968a7aa48527 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-14T09:14:58,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=A 2024-11-14T09:14:58,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:14:58,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=B 2024-11-14T09:14:58,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:14:58,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=C 2024-11-14T09:14:58,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:14:58,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on 8664731a05c52a198640968a7aa48527 2024-11-14T09:14:58,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. as already flushing 2024-11-14T09:14:58,635 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8664731a05c52a198640968a7aa48527#A#compaction#253 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:14:58,635 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/3b1cfcc863f4439d8c1dcc514fcce097 is 50, key is test_row_0/A:col10/1731575697348/Put/seqid=0 2024-11-14T09:14:58,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/ba6b6324f13842f1931bd781fc8c725d is 50, key is test_row_0/A:col10/1731575697515/Put/seqid=0 2024-11-14T09:14:58,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742125_1301 (size=12697) 2024-11-14T09:14:58,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:58,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58142 deadline: 1731575758645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:58,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:58,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58168 deadline: 1731575758645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:58,649 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_2' in 8664731a05c52a198640968a7aa48527 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8234 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:14:58,650 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_2' in 8664731a05c52a198640968a7aa48527 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8236 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:14:58,650 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:58,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:58,650 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:58,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:58,650 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:58,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:58,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742124_1300 (size=12697) 2024-11-14T09:14:58,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:58,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58122 deadline: 1731575758649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:58,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:58,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58184 deadline: 1731575758649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:58,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:58,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:58,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:58,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:58,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:58,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:58,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742126_1302 (size=12151) 2024-11-14T09:14:58,661 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/b8fb581b68d54d6789ed86067acb29af as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/b8fb581b68d54d6789ed86067acb29af 2024-11-14T09:14:58,666 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8664731a05c52a198640968a7aa48527/B of 8664731a05c52a198640968a7aa48527 into b8fb581b68d54d6789ed86067acb29af(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:14:58,666 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8664731a05c52a198640968a7aa48527: 2024-11-14T09:14:58,666 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., storeName=8664731a05c52a198640968a7aa48527/B, priority=13, startTime=1731575698615; duration=0sec 2024-11-14T09:14:58,666 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:14:58,666 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8664731a05c52a198640968a7aa48527:B 2024-11-14T09:14:58,666 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:14:58,667 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:14:58,667 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): 8664731a05c52a198640968a7aa48527/C is initiating minor compaction (all files) 2024-11-14T09:14:58,667 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8664731a05c52a198640968a7aa48527/C in TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:58,667 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/e245f91696334f6fac42c55910e4fdcb, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/dc22d4eaa5114ef1986204aaaf7df1ea, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/26534f0444824557b8192887022d3274] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp, totalSize=36.0 K 2024-11-14T09:14:58,668 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting e245f91696334f6fac42c55910e4fdcb, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731575693926 2024-11-14T09:14:58,668 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting dc22d4eaa5114ef1986204aaaf7df1ea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731575695168 2024-11-14T09:14:58,668 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 26534f0444824557b8192887022d3274, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731575697341 2024-11-14T09:14:58,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:58,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731575758665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:58,669 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:58,669 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:58,669 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:58,678 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8664731a05c52a198640968a7aa48527#C#compaction#255 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:14:58,679 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/ae1ea5bbf77d4d8db014f3b642d13310 is 50, key is test_row_0/C:col10/1731575697348/Put/seqid=0 2024-11-14T09:14:58,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742127_1303 (size=12697) 2024-11-14T09:14:58,688 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/ae1ea5bbf77d4d8db014f3b642d13310 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/ae1ea5bbf77d4d8db014f3b642d13310 2024-11-14T09:14:58,693 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8664731a05c52a198640968a7aa48527/C of 8664731a05c52a198640968a7aa48527 into ae1ea5bbf77d4d8db014f3b642d13310(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:14:58,693 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8664731a05c52a198640968a7aa48527: 2024-11-14T09:14:58,693 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., storeName=8664731a05c52a198640968a7aa48527/C, priority=13, startTime=1731575698615; duration=0sec 2024-11-14T09:14:58,693 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:58,693 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8664731a05c52a198640968a7aa48527:C 2024-11-14T09:14:58,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:58,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58184 deadline: 1731575758765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:58,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:58,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:58,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58122 deadline: 1731575758765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:58,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:58,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:58,768 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:58,768 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:58,768 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:58,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T09:14:58,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:58,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58184 deadline: 1731575758975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:58,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:58,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:58,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58122 deadline: 1731575758975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:58,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:58,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:58,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:58,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:58,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:59,048 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/3b1cfcc863f4439d8c1dcc514fcce097 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/3b1cfcc863f4439d8c1dcc514fcce097 2024-11-14T09:14:59,053 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8664731a05c52a198640968a7aa48527/A of 8664731a05c52a198640968a7aa48527 into 3b1cfcc863f4439d8c1dcc514fcce097(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:14:59,053 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8664731a05c52a198640968a7aa48527: 2024-11-14T09:14:59,053 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., storeName=8664731a05c52a198640968a7aa48527/A, priority=13, startTime=1731575698615; duration=0sec 2024-11-14T09:14:59,053 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:14:59,053 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8664731a05c52a198640968a7aa48527:A 2024-11-14T09:14:59,058 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/ba6b6324f13842f1931bd781fc8c725d 2024-11-14T09:14:59,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/e970e565340a4b5cb9a43ff294869ecd is 50, key is test_row_0/B:col10/1731575697515/Put/seqid=0 2024-11-14T09:14:59,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742128_1304 (size=12151) 2024-11-14T09:14:59,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:59,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58184 deadline: 1731575759285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:59,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:59,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58122 deadline: 1731575759285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:59,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:59,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:59,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:59,288 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:59,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:59,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:59,470 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/e970e565340a4b5cb9a43ff294869ecd 2024-11-14T09:14:59,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/37fd770cd174471185df0fbc8d974fcf is 50, key is test_row_0/C:col10/1731575697515/Put/seqid=0 2024-11-14T09:14:59,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742129_1305 (size=12151) 2024-11-14T09:14:59,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:59,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731575759675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:59,678 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_2' in 8664731a05c52a198640968a7aa48527 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:14:59,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:59,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:59,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:59,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:59,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58184 deadline: 1731575759795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:59,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:14:59,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:59,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:59,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58122 deadline: 1731575759795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:14:59,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:59,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:59,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:14:59,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:14:59,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T09:14:59,882 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/37fd770cd174471185df0fbc8d974fcf 2024-11-14T09:14:59,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/ba6b6324f13842f1931bd781fc8c725d as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/ba6b6324f13842f1931bd781fc8c725d 2024-11-14T09:14:59,891 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/ba6b6324f13842f1931bd781fc8c725d, entries=150, sequenceid=250, filesize=11.9 K 2024-11-14T09:14:59,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/e970e565340a4b5cb9a43ff294869ecd as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/e970e565340a4b5cb9a43ff294869ecd 2024-11-14T09:14:59,895 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/e970e565340a4b5cb9a43ff294869ecd, entries=150, sequenceid=250, filesize=11.9 K 2024-11-14T09:14:59,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/37fd770cd174471185df0fbc8d974fcf as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/37fd770cd174471185df0fbc8d974fcf 2024-11-14T09:14:59,900 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/37fd770cd174471185df0fbc8d974fcf, entries=150, sequenceid=250, filesize=11.9 K 2024-11-14T09:14:59,900 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 8664731a05c52a198640968a7aa48527 in 1274ms, sequenceid=250, compaction requested=false 2024-11-14T09:14:59,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 8664731a05c52a198640968a7aa48527: 2024-11-14T09:14:59,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:14:59,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-14T09:14:59,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-14T09:14:59,904 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-14T09:14:59,904 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1940 sec 2024-11-14T09:14:59,905 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 2.1990 sec 2024-11-14T09:15:00,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on 8664731a05c52a198640968a7aa48527 2024-11-14T09:15:00,807 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8664731a05c52a198640968a7aa48527 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-14T09:15:00,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=A 2024-11-14T09:15:00,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:15:00,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=B 2024-11-14T09:15:00,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:15:00,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=C 2024-11-14T09:15:00,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:15:00,814 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/ca30e32004d346a8b030c0428c3ae5f4 is 50, key is test_row_1/A:col10/1731575700805/Put/seqid=0 2024-11-14T09:15:00,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742130_1306 (size=9857) 2024-11-14T09:15:00,820 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/ca30e32004d346a8b030c0428c3ae5f4 2024-11-14T09:15:00,829 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/0529993b8b5b411797507d94240f6d21 is 50, key is test_row_1/B:col10/1731575700805/Put/seqid=0 2024-11-14T09:15:00,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742131_1307 (size=9857) 2024-11-14T09:15:00,838 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/0529993b8b5b411797507d94240f6d21 2024-11-14T09:15:00,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/e3c8beb30fb94fa1877c3e3fd686334b is 50, key is test_row_1/C:col10/1731575700805/Put/seqid=0 2024-11-14T09:15:00,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742132_1308 (size=9857) 2024-11-14T09:15:00,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:00,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58122 deadline: 1731575760852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:00,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:00,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:00,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:00,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731575700836, max timestamp=1731575700853Num uniques 859; 2024-11-14T09:15:00,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731575700836, max timestamp=1731575700853Num uniques 859; 2024-11-14T09:15:00,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:15:00,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:15:00,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731575700836, max timestamp=1731575700853Num uniques 859; 2024-11-14T09:15:00,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:15:00,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:00,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58184 deadline: 1731575760930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:00,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:00,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:00,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:00,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:00,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58122 deadline: 1731575760964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:00,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:00,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:00,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:01,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:01,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58184 deadline: 1731575761045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:01,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:01,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:01,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:01,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:01,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58122 deadline: 1731575761175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:01,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:01,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:01,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:01,253 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/e3c8beb30fb94fa1877c3e3fd686334b 2024-11-14T09:15:01,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:01,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58184 deadline: 1731575761255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:01,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:01,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:01,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:01,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/ca30e32004d346a8b030c0428c3ae5f4 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/ca30e32004d346a8b030c0428c3ae5f4 2024-11-14T09:15:01,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/ca30e32004d346a8b030c0428c3ae5f4, entries=100, sequenceid=264, filesize=9.6 K 2024-11-14T09:15:01,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/0529993b8b5b411797507d94240f6d21 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/0529993b8b5b411797507d94240f6d21 2024-11-14T09:15:01,275 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/0529993b8b5b411797507d94240f6d21, entries=100, sequenceid=264, filesize=9.6 K 2024-11-14T09:15:01,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/e3c8beb30fb94fa1877c3e3fd686334b as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/e3c8beb30fb94fa1877c3e3fd686334b 2024-11-14T09:15:01,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/e3c8beb30fb94fa1877c3e3fd686334b, entries=100, sequenceid=264, filesize=9.6 K 2024-11-14T09:15:01,282 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 8664731a05c52a198640968a7aa48527 in 475ms, sequenceid=264, compaction requested=true 2024-11-14T09:15:01,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8664731a05c52a198640968a7aa48527: 2024-11-14T09:15:01,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8664731a05c52a198640968a7aa48527:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:15:01,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:15:01,282 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:15:01,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8664731a05c52a198640968a7aa48527:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:15:01,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:15:01,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8664731a05c52a198640968a7aa48527:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:15:01,282 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:15:01,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:15:01,283 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34705 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:15:01,283 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34705 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:15:01,284 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): 8664731a05c52a198640968a7aa48527/A is initiating minor compaction (all files) 2024-11-14T09:15:01,284 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): 8664731a05c52a198640968a7aa48527/B is initiating minor compaction (all files) 2024-11-14T09:15:01,284 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8664731a05c52a198640968a7aa48527/B in TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:15:01,284 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8664731a05c52a198640968a7aa48527/A in TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:15:01,284 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/3b1cfcc863f4439d8c1dcc514fcce097, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/ba6b6324f13842f1931bd781fc8c725d, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/ca30e32004d346a8b030c0428c3ae5f4] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp, totalSize=33.9 K 2024-11-14T09:15:01,284 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/b8fb581b68d54d6789ed86067acb29af, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/e970e565340a4b5cb9a43ff294869ecd, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/0529993b8b5b411797507d94240f6d21] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp, totalSize=33.9 K 2024-11-14T09:15:01,284 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting b8fb581b68d54d6789ed86067acb29af, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731575697341 2024-11-14T09:15:01,284 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b1cfcc863f4439d8c1dcc514fcce097, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731575697341 2024-11-14T09:15:01,285 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting e970e565340a4b5cb9a43ff294869ecd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731575697508 2024-11-14T09:15:01,285 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba6b6324f13842f1931bd781fc8c725d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731575697508 2024-11-14T09:15:01,285 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 0529993b8b5b411797507d94240f6d21, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731575698646 2024-11-14T09:15:01,285 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca30e32004d346a8b030c0428c3ae5f4, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731575698646 2024-11-14T09:15:01,299 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8664731a05c52a198640968a7aa48527#A#compaction#261 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:15:01,299 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/c4d4af63ca9e4568a0024d498871c06f is 50, key is test_row_0/A:col10/1731575697515/Put/seqid=0 2024-11-14T09:15:01,302 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8664731a05c52a198640968a7aa48527#B#compaction#262 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:15:01,303 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/183c8239357d4beda4e2a270eaa93410 is 50, key is test_row_0/B:col10/1731575697515/Put/seqid=0 2024-11-14T09:15:01,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742133_1309 (size=12899) 2024-11-14T09:15:01,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742134_1310 (size=12899) 2024-11-14T09:15:01,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on 8664731a05c52a198640968a7aa48527 2024-11-14T09:15:01,490 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8664731a05c52a198640968a7aa48527 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-14T09:15:01,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=A 2024-11-14T09:15:01,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:15:01,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=B 2024-11-14T09:15:01,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:15:01,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=C 2024-11-14T09:15:01,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:15:01,497 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/824215931234457db1a42e4a04433c7e is 50, key is test_row_0/A:col10/1731575700922/Put/seqid=0 2024-11-14T09:15:01,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742135_1311 (size=12301) 2024-11-14T09:15:01,513 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/824215931234457db1a42e4a04433c7e 2024-11-14T09:15:01,528 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/f9465e73e68c4d1bb8953155fbd06d2c is 50, key is test_row_0/B:col10/1731575700922/Put/seqid=0 2024-11-14T09:15:01,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:01,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58122 deadline: 1731575761541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:01,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:01,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:01,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:01,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742136_1312 (size=12301) 2024-11-14T09:15:01,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:01,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58184 deadline: 1731575761565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:01,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:01,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:01,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:01,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:01,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58122 deadline: 1731575761654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:01,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:01,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:01,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:01,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:01,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731575761685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:01,688 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_2' in 8664731a05c52a198640968a7aa48527 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:15:01,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:01,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:01,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:01,722 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/c4d4af63ca9e4568a0024d498871c06f as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/c4d4af63ca9e4568a0024d498871c06f 2024-11-14T09:15:01,730 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/183c8239357d4beda4e2a270eaa93410 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/183c8239357d4beda4e2a270eaa93410 2024-11-14T09:15:01,731 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8664731a05c52a198640968a7aa48527/A of 8664731a05c52a198640968a7aa48527 into c4d4af63ca9e4568a0024d498871c06f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:15:01,731 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8664731a05c52a198640968a7aa48527: 2024-11-14T09:15:01,731 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., storeName=8664731a05c52a198640968a7aa48527/A, priority=13, startTime=1731575701282; duration=0sec 2024-11-14T09:15:01,731 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:15:01,731 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8664731a05c52a198640968a7aa48527:A 2024-11-14T09:15:01,731 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:15:01,742 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34705 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:15:01,742 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): 8664731a05c52a198640968a7aa48527/C is initiating minor compaction (all files) 2024-11-14T09:15:01,742 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8664731a05c52a198640968a7aa48527/C in TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:15:01,742 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/ae1ea5bbf77d4d8db014f3b642d13310, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/37fd770cd174471185df0fbc8d974fcf, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/e3c8beb30fb94fa1877c3e3fd686334b] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp, totalSize=33.9 K 2024-11-14T09:15:01,742 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae1ea5bbf77d4d8db014f3b642d13310, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731575697341 2024-11-14T09:15:01,743 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37fd770cd174471185df0fbc8d974fcf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731575697508 2024-11-14T09:15:01,745 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8664731a05c52a198640968a7aa48527/B of 8664731a05c52a198640968a7aa48527 into 183c8239357d4beda4e2a270eaa93410(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:15:01,745 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8664731a05c52a198640968a7aa48527: 2024-11-14T09:15:01,746 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., storeName=8664731a05c52a198640968a7aa48527/B, priority=13, startTime=1731575701282; duration=0sec 2024-11-14T09:15:01,746 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:15:01,746 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8664731a05c52a198640968a7aa48527:B 2024-11-14T09:15:01,752 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3c8beb30fb94fa1877c3e3fd686334b, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731575698646 2024-11-14T09:15:01,760 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8664731a05c52a198640968a7aa48527#C#compaction#265 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:15:01,761 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/f402d66f9ba342f1ab9379d97aa578d2 is 50, key is test_row_0/C:col10/1731575697515/Put/seqid=0 2024-11-14T09:15:01,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742137_1313 (size=12899) 2024-11-14T09:15:01,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:01,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58122 deadline: 1731575761865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:01,871 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:01,871 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:01,871 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:01,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T09:15:01,885 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:15:01,886 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:15:01,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-14T09:15:01,889 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:15:01,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-14T09:15:01,890 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:15:01,890 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:15:01,950 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/f9465e73e68c4d1bb8953155fbd06d2c 2024-11-14T09:15:01,965 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/c242630cdfb64e129db1c994375549da is 50, key is test_row_0/C:col10/1731575700922/Put/seqid=0 2024-11-14T09:15:01,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742138_1314 (size=12301) 2024-11-14T09:15:01,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-14T09:15:02,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-14T09:15:02,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:15:02,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. as already flushing 2024-11-14T09:15:02,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:15:02,044 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:02,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:02,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:02,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:02,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58184 deadline: 1731575762075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:02,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:02,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:02,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:02,175 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/f402d66f9ba342f1ab9379d97aa578d2 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/f402d66f9ba342f1ab9379d97aa578d2 2024-11-14T09:15:02,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:02,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58122 deadline: 1731575762175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:02,177 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:02,177 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:02,177 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:02,181 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8664731a05c52a198640968a7aa48527/C of 8664731a05c52a198640968a7aa48527 into f402d66f9ba342f1ab9379d97aa578d2(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:15:02,181 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8664731a05c52a198640968a7aa48527: 2024-11-14T09:15:02,181 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., storeName=8664731a05c52a198640968a7aa48527/C, priority=13, startTime=1731575701282; duration=0sec 2024-11-14T09:15:02,181 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:15:02,181 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8664731a05c52a198640968a7aa48527:C 2024-11-14T09:15:02,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-14T09:15:02,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:15:02,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. as already flushing 2024-11-14T09:15:02,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:15:02,197 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:02,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:02,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:02,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-14T09:15:02,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-14T09:15:02,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:15:02,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. as already flushing 2024-11-14T09:15:02,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:15:02,350 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:02,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:02,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:02,372 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/c242630cdfb64e129db1c994375549da 2024-11-14T09:15:02,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/824215931234457db1a42e4a04433c7e as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/824215931234457db1a42e4a04433c7e 2024-11-14T09:15:02,380 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/824215931234457db1a42e4a04433c7e, entries=150, sequenceid=301, filesize=12.0 K 2024-11-14T09:15:02,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/f9465e73e68c4d1bb8953155fbd06d2c as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/f9465e73e68c4d1bb8953155fbd06d2c 2024-11-14T09:15:02,384 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/f9465e73e68c4d1bb8953155fbd06d2c, entries=150, sequenceid=301, filesize=12.0 K 2024-11-14T09:15:02,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/c242630cdfb64e129db1c994375549da as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/c242630cdfb64e129db1c994375549da 2024-11-14T09:15:02,388 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/c242630cdfb64e129db1c994375549da, entries=150, sequenceid=301, filesize=12.0 K 2024-11-14T09:15:02,389 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 8664731a05c52a198640968a7aa48527 in 899ms, sequenceid=301, compaction requested=false 2024-11-14T09:15:02,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8664731a05c52a198640968a7aa48527: 2024-11-14T09:15:02,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-14T09:15:02,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:15:02,504 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 8664731a05c52a198640968a7aa48527 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T09:15:02,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=A 2024-11-14T09:15:02,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:15:02,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=B 2024-11-14T09:15:02,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:15:02,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=C 2024-11-14T09:15:02,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:15:02,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/f6e6948eca374994b7419eb312eb974a is 50, key is test_row_0/A:col10/1731575701510/Put/seqid=0 2024-11-14T09:15:02,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742139_1315 (size=12301) 2024-11-14T09:15:02,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-14T09:15:02,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on 8664731a05c52a198640968a7aa48527 2024-11-14T09:15:02,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. as already flushing 2024-11-14T09:15:02,767 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731575702740, max timestamp=1731575702764Num uniques 859; 2024-11-14T09:15:02,767 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:15:02,767 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731575702740, max timestamp=1731575702764Num uniques 859; 2024-11-14T09:15:02,767 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:15:02,767 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731575702740, max timestamp=1731575702764Num uniques 859; 2024-11-14T09:15:02,768 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:15:02,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:02,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58122 deadline: 1731575762837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:02,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:02,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:02,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:02,914 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/f6e6948eca374994b7419eb312eb974a 2024-11-14T09:15:02,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/d4dc39b5ee6942e6bc43413d7dcd63a1 is 50, key is test_row_0/B:col10/1731575701510/Put/seqid=0 2024-11-14T09:15:02,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742140_1316 (size=12301) 2024-11-14T09:15:02,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:02,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58122 deadline: 1731575762945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:02,948 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:02,948 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:02,948 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:03,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-14T09:15:03,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:03,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58184 deadline: 1731575763085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:03,089 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_1' in 8664731a05c52a198640968a7aa48527 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:15:03,090 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:03,090 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:03,090 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:03,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:03,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58122 deadline: 1731575763155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:03,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:03,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:03,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:03,326 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/d4dc39b5ee6942e6bc43413d7dcd63a1 2024-11-14T09:15:03,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/6b7684187d8444738146c5bed424fe69 is 50, key is test_row_0/C:col10/1731575701510/Put/seqid=0 2024-11-14T09:15:03,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742141_1317 (size=12301) 2024-11-14T09:15:03,340 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/6b7684187d8444738146c5bed424fe69 2024-11-14T09:15:03,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/f6e6948eca374994b7419eb312eb974a as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/f6e6948eca374994b7419eb312eb974a 2024-11-14T09:15:03,351 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/f6e6948eca374994b7419eb312eb974a, entries=150, sequenceid=314, filesize=12.0 K 2024-11-14T09:15:03,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/d4dc39b5ee6942e6bc43413d7dcd63a1 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/d4dc39b5ee6942e6bc43413d7dcd63a1 2024-11-14T09:15:03,356 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/d4dc39b5ee6942e6bc43413d7dcd63a1, entries=150, sequenceid=314, filesize=12.0 K 2024-11-14T09:15:03,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/6b7684187d8444738146c5bed424fe69 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/6b7684187d8444738146c5bed424fe69 2024-11-14T09:15:03,362 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/6b7684187d8444738146c5bed424fe69, entries=150, sequenceid=314, filesize=12.0 K 2024-11-14T09:15:03,363 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 8664731a05c52a198640968a7aa48527 in 860ms, sequenceid=314, compaction requested=true 2024-11-14T09:15:03,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 8664731a05c52a198640968a7aa48527: 2024-11-14T09:15:03,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:15:03,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-14T09:15:03,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-14T09:15:03,368 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-14T09:15:03,368 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4760 sec 2024-11-14T09:15:03,370 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.4830 sec 2024-11-14T09:15:03,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on 8664731a05c52a198640968a7aa48527 2024-11-14T09:15:03,469 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8664731a05c52a198640968a7aa48527 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-14T09:15:03,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=A 2024-11-14T09:15:03,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-14T09:15:03,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=B 2024-11-14T09:15:03,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-14T09:15:03,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=C 2024-11-14T09:15:03,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-14T09:15:03,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/fc08047c6b63468d9de846d9d1400d41 is 50, key is test_row_0/A:col10/1731575702835/Put/seqid=0 2024-11-14T09:15:03,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742142_1318 (size=14741) 2024-11-14T09:15:03,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:03,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58122 deadline: 1731575763506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:03,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:03,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:03,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:03,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:03,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58122 deadline: 1731575763615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:03,618 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:03,618 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:03,618 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:03,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:03,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58122 deadline: 1731575763825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:03,829 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:03,830 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:03,830 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:03,881 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/fc08047c6b63468d9de846d9d1400d41 2024-11-14T09:15:03,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/1f6d7edb8253458b9137776c4462ed0b is 50, key is test_row_0/B:col10/1731575702835/Put/seqid=0 2024-11-14T09:15:03,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742143_1319 (size=12301) 2024-11-14T09:15:04,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-14T09:15:04,035 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:15:04,035 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:15:04,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-14T09:15:04,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-14T09:15:04,037 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:15:04,038 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:15:04,038 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:15:04,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:04,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58122 deadline: 1731575764135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:04,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:04,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:04,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:04,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-14T09:15:04,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-14T09:15:04,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:15:04,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. as already flushing 2024-11-14T09:15:04,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:15:04,190 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:04,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:04,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:04,293 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/1f6d7edb8253458b9137776c4462ed0b 2024-11-14T09:15:04,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/1eab68bb11134d14a4c09ba323b60440 is 50, key is test_row_0/C:col10/1731575702835/Put/seqid=0 2024-11-14T09:15:04,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742144_1320 (size=12301) 2024-11-14T09:15:04,307 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/1eab68bb11134d14a4c09ba323b60440 2024-11-14T09:15:04,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/fc08047c6b63468d9de846d9d1400d41 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/fc08047c6b63468d9de846d9d1400d41 2024-11-14T09:15:04,316 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/fc08047c6b63468d9de846d9d1400d41, entries=200, sequenceid=352, filesize=14.4 K 2024-11-14T09:15:04,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/1f6d7edb8253458b9137776c4462ed0b as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/1f6d7edb8253458b9137776c4462ed0b 2024-11-14T09:15:04,320 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/1f6d7edb8253458b9137776c4462ed0b, entries=150, sequenceid=352, filesize=12.0 K 2024-11-14T09:15:04,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/1eab68bb11134d14a4c09ba323b60440 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/1eab68bb11134d14a4c09ba323b60440 2024-11-14T09:15:04,325 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/1eab68bb11134d14a4c09ba323b60440, entries=150, sequenceid=352, filesize=12.0 K 2024-11-14T09:15:04,326 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 8664731a05c52a198640968a7aa48527 in 857ms, sequenceid=352, compaction requested=true 2024-11-14T09:15:04,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8664731a05c52a198640968a7aa48527: 2024-11-14T09:15:04,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8664731a05c52a198640968a7aa48527:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:15:04,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:15:04,326 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:15:04,326 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:15:04,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8664731a05c52a198640968a7aa48527:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:15:04,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:15:04,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8664731a05c52a198640968a7aa48527:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:15:04,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:15:04,330 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52242 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:15:04,330 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49802 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:15:04,330 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): 8664731a05c52a198640968a7aa48527/A is initiating minor compaction (all files) 2024-11-14T09:15:04,330 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): 8664731a05c52a198640968a7aa48527/B is initiating minor compaction (all files) 2024-11-14T09:15:04,330 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8664731a05c52a198640968a7aa48527/B in TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:15:04,330 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8664731a05c52a198640968a7aa48527/A in TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:15:04,330 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/183c8239357d4beda4e2a270eaa93410, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/f9465e73e68c4d1bb8953155fbd06d2c, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/d4dc39b5ee6942e6bc43413d7dcd63a1, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/1f6d7edb8253458b9137776c4462ed0b] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp, totalSize=48.6 K 2024-11-14T09:15:04,330 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/c4d4af63ca9e4568a0024d498871c06f, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/824215931234457db1a42e4a04433c7e, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/f6e6948eca374994b7419eb312eb974a, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/fc08047c6b63468d9de846d9d1400d41] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp, totalSize=51.0 K 2024-11-14T09:15:04,331 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 183c8239357d4beda4e2a270eaa93410, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731575697515 2024-11-14T09:15:04,331 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4d4af63ca9e4568a0024d498871c06f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731575697515 2024-11-14T09:15:04,331 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 824215931234457db1a42e4a04433c7e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731575700922 2024-11-14T09:15:04,332 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6e6948eca374994b7419eb312eb974a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731575701504 2024-11-14T09:15:04,332 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc08047c6b63468d9de846d9d1400d41, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731575702820 2024-11-14T09:15:04,333 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting f9465e73e68c4d1bb8953155fbd06d2c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731575700922 2024-11-14T09:15:04,334 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting d4dc39b5ee6942e6bc43413d7dcd63a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731575701504 2024-11-14T09:15:04,334 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f6d7edb8253458b9137776c4462ed0b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731575702820 2024-11-14T09:15:04,343 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-14T09:15:04,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:15:04,344 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 8664731a05c52a198640968a7aa48527 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-14T09:15:04,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=A 2024-11-14T09:15:04,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:15:04,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=B 2024-11-14T09:15:04,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:15:04,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=C 2024-11-14T09:15:04,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:15:04,346 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8664731a05c52a198640968a7aa48527#A#compaction#273 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:15:04,347 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/0f839c0d21d0462f9817a0dd1a003b12 is 50, key is test_row_0/A:col10/1731575702835/Put/seqid=0 2024-11-14T09:15:04,355 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8664731a05c52a198640968a7aa48527#B#compaction#274 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:15:04,356 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/b9a2d1bc22b74c93a6fc53061a41afbe is 50, key is test_row_0/B:col10/1731575702835/Put/seqid=0 2024-11-14T09:15:04,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-14T09:15:04,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/e9d02b6889554183af684b7115a1e8be is 50, key is test_row_0/A:col10/1731575703504/Put/seqid=0 2024-11-14T09:15:04,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742145_1321 (size=13085) 2024-11-14T09:15:04,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742146_1322 (size=13085) 2024-11-14T09:15:04,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742147_1323 (size=9857) 2024-11-14T09:15:04,417 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/e9d02b6889554183af684b7115a1e8be 2024-11-14T09:15:04,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/de2f0479f83f4d7692ea4df2f3c418d4 is 50, key is test_row_0/B:col10/1731575703504/Put/seqid=0 2024-11-14T09:15:04,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742148_1324 (size=9857) 2024-11-14T09:15:04,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on 8664731a05c52a198640968a7aa48527 2024-11-14T09:15:04,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. as already flushing 2024-11-14T09:15:04,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-14T09:15:04,722 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=109, cellSize=4985, totalHeapSize=11576, min timestamp=1731575704711, max timestamp=1731575704720Num uniques 859; 2024-11-14T09:15:04,722 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=109, cellSize=4985, totalHeapSize=11576, min timestamp=1731575704711, max timestamp=1731575704720Num uniques 859; 2024-11-14T09:15:04,722 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=109, cellSize=4985, totalHeapSize=11576, min timestamp=1731575704711, max timestamp=1731575704720Num uniques 859; 2024-11-14T09:15:04,722 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=4985, suffix heap size=117064,new segment heap size=11576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=109 2024-11-14T09:15:04,722 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=4985, suffix heap size=117064,new segment heap size=11576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=109 2024-11-14T09:15:04,722 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=4985, suffix heap size=117064,new segment heap size=11576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=109 2024-11-14T09:15:04,800 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/0f839c0d21d0462f9817a0dd1a003b12 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/0f839c0d21d0462f9817a0dd1a003b12 2024-11-14T09:15:04,806 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8664731a05c52a198640968a7aa48527/A of 8664731a05c52a198640968a7aa48527 into 0f839c0d21d0462f9817a0dd1a003b12(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:15:04,806 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8664731a05c52a198640968a7aa48527: 2024-11-14T09:15:04,806 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., storeName=8664731a05c52a198640968a7aa48527/A, priority=12, startTime=1731575704326; duration=0sec 2024-11-14T09:15:04,806 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:15:04,806 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8664731a05c52a198640968a7aa48527:A 2024-11-14T09:15:04,806 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:15:04,807 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49802 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:15:04,807 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/b9a2d1bc22b74c93a6fc53061a41afbe as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/b9a2d1bc22b74c93a6fc53061a41afbe 2024-11-14T09:15:04,807 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): 8664731a05c52a198640968a7aa48527/C is initiating minor compaction (all files) 2024-11-14T09:15:04,807 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8664731a05c52a198640968a7aa48527/C in TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:15:04,808 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/f402d66f9ba342f1ab9379d97aa578d2, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/c242630cdfb64e129db1c994375549da, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/6b7684187d8444738146c5bed424fe69, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/1eab68bb11134d14a4c09ba323b60440] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp, totalSize=48.6 K 2024-11-14T09:15:04,808 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting f402d66f9ba342f1ab9379d97aa578d2, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731575697515 2024-11-14T09:15:04,808 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting c242630cdfb64e129db1c994375549da, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731575700922 2024-11-14T09:15:04,809 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b7684187d8444738146c5bed424fe69, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731575701504 2024-11-14T09:15:04,809 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1eab68bb11134d14a4c09ba323b60440, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731575702820 2024-11-14T09:15:04,817 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8664731a05c52a198640968a7aa48527/B of 8664731a05c52a198640968a7aa48527 into b9a2d1bc22b74c93a6fc53061a41afbe(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:15:04,817 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8664731a05c52a198640968a7aa48527: 2024-11-14T09:15:04,817 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., storeName=8664731a05c52a198640968a7aa48527/B, priority=12, startTime=1731575704326; duration=0sec 2024-11-14T09:15:04,817 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:15:04,817 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8664731a05c52a198640968a7aa48527:B 2024-11-14T09:15:04,820 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731575704778, max timestamp=1731575704816Num uniques 968; 2024-11-14T09:15:04,820 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=44328, new segment data size=7688, suffix heap size=128640,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=968, new segment cells count=168 2024-11-14T09:15:04,820 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731575704778, max timestamp=1731575704816Num uniques 968; 2024-11-14T09:15:04,820 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731575704778, max timestamp=1731575704816Num uniques 968; 2024-11-14T09:15:04,820 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=44328, new segment data size=7688, suffix heap size=128640,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=968, new segment cells count=168 2024-11-14T09:15:04,820 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=44328, new segment data size=7688, suffix heap size=128640,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=968, new segment cells count=168 2024-11-14T09:15:04,825 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8664731a05c52a198640968a7aa48527#C#compaction#277 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:15:04,825 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/4184d35c74ff4981977979c6e96d9aa6 is 50, key is test_row_0/C:col10/1731575702835/Put/seqid=0 2024-11-14T09:15:04,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742149_1325 (size=13085) 2024-11-14T09:15:04,854 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/de2f0479f83f4d7692ea4df2f3c418d4 2024-11-14T09:15:04,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/d2602f7f5ed94676afa91f3a1812fbe3 is 50, key is test_row_0/C:col10/1731575703504/Put/seqid=0 2024-11-14T09:15:04,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742150_1326 (size=9857) 2024-11-14T09:15:04,865 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/d2602f7f5ed94676afa91f3a1812fbe3 2024-11-14T09:15:04,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/e9d02b6889554183af684b7115a1e8be as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/e9d02b6889554183af684b7115a1e8be 2024-11-14T09:15:04,873 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/e9d02b6889554183af684b7115a1e8be, entries=100, sequenceid=361, filesize=9.6 K 2024-11-14T09:15:04,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/de2f0479f83f4d7692ea4df2f3c418d4 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/de2f0479f83f4d7692ea4df2f3c418d4 2024-11-14T09:15:04,878 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/de2f0479f83f4d7692ea4df2f3c418d4, entries=100, sequenceid=361, filesize=9.6 K 2024-11-14T09:15:04,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/d2602f7f5ed94676afa91f3a1812fbe3 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/d2602f7f5ed94676afa91f3a1812fbe3 2024-11-14T09:15:04,883 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/d2602f7f5ed94676afa91f3a1812fbe3, entries=100, sequenceid=361, filesize=9.6 K 2024-11-14T09:15:04,883 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=120.76 KB/123660 for 8664731a05c52a198640968a7aa48527 in 539ms, sequenceid=361, compaction requested=false 2024-11-14T09:15:04,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 8664731a05c52a198640968a7aa48527: 2024-11-14T09:15:04,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:15:04,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-14T09:15:04,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-14T09:15:04,888 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-14T09:15:04,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on 8664731a05c52a198640968a7aa48527 2024-11-14T09:15:04,888 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 847 msec 2024-11-14T09:15:04,888 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8664731a05c52a198640968a7aa48527 3/3 column families, dataSize=127.47 KB heapSize=364.22 KB 2024-11-14T09:15:04,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=A 2024-11-14T09:15:04,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:15:04,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=B 2024-11-14T09:15:04,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:15:04,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=C 2024-11-14T09:15:04,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:15:04,890 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 854 msec 2024-11-14T09:15:04,893 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/34adbd921d6d49fe9f2c07e453be155a is 50, key is test_row_0/A:col10/1731575704881/Put/seqid=0 2024-11-14T09:15:04,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742151_1327 (size=14741) 2024-11-14T09:15:04,898 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/34adbd921d6d49fe9f2c07e453be155a 2024-11-14T09:15:04,906 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/d8ca9864a3bc4f99badfeab27a91862a is 50, key is test_row_0/B:col10/1731575704881/Put/seqid=0 2024-11-14T09:15:04,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742152_1328 (size=12301) 2024-11-14T09:15:04,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:04,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58122 deadline: 1731575764932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:04,937 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:04,937 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:04,937 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:05,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:05,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58122 deadline: 1731575765045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:05,049 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:05,049 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:05,049 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:05,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:05,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58184 deadline: 1731575765094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:05,098 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_1' in 8664731a05c52a198640968a7aa48527 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:15:05,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:05,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:05,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:05,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-14T09:15:05,176 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:15:05,176 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:15:05,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-14T09:15:05,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-14T09:15:05,178 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:15:05,179 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:15:05,179 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:15:05,235 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/4184d35c74ff4981977979c6e96d9aa6 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/4184d35c74ff4981977979c6e96d9aa6 2024-11-14T09:15:05,240 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8664731a05c52a198640968a7aa48527/C of 8664731a05c52a198640968a7aa48527 into 4184d35c74ff4981977979c6e96d9aa6(size=12.8 K), total size for store is 22.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:15:05,240 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8664731a05c52a198640968a7aa48527: 2024-11-14T09:15:05,240 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., storeName=8664731a05c52a198640968a7aa48527/C, priority=12, startTime=1731575704326; duration=0sec 2024-11-14T09:15:05,240 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:15:05,240 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8664731a05c52a198640968a7aa48527:C 2024-11-14T09:15:05,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:05,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58122 deadline: 1731575765255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:05,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:05,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:05,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:05,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-14T09:15:05,312 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/d8ca9864a3bc4f99badfeab27a91862a 2024-11-14T09:15:05,320 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/56f7a5ad05fc40ce8546db5b151f350a is 50, key is test_row_0/C:col10/1731575704881/Put/seqid=0 2024-11-14T09:15:05,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742153_1329 (size=12301) 2024-11-14T09:15:05,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-14T09:15:05,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:15:05,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. as already flushing 2024-11-14T09:15:05,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:15:05,331 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:05,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:05,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:05,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-14T09:15:05,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:15:05,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. as already flushing 2024-11-14T09:15:05,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:15:05,484 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:05,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:05,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:05,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-14T09:15:05,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:05,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58122 deadline: 1731575765565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:05,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:05,566 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:05,566 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:05,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-14T09:15:05,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:15:05,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. as already flushing 2024-11-14T09:15:05,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:15:05,638 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:05,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:05,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:05,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:05,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731575765705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:05,709 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_2' in 8664731a05c52a198640968a7aa48527 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:15:05,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:05,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:05,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:05,724 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/56f7a5ad05fc40ce8546db5b151f350a 2024-11-14T09:15:05,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/34adbd921d6d49fe9f2c07e453be155a as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/34adbd921d6d49fe9f2c07e453be155a 2024-11-14T09:15:05,735 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/34adbd921d6d49fe9f2c07e453be155a, entries=200, sequenceid=416, filesize=14.4 K 2024-11-14T09:15:05,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/d8ca9864a3bc4f99badfeab27a91862a as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/d8ca9864a3bc4f99badfeab27a91862a 2024-11-14T09:15:05,739 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/d8ca9864a3bc4f99badfeab27a91862a, entries=150, sequenceid=416, filesize=12.0 K 2024-11-14T09:15:05,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/56f7a5ad05fc40ce8546db5b151f350a as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/56f7a5ad05fc40ce8546db5b151f350a 2024-11-14T09:15:05,745 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/56f7a5ad05fc40ce8546db5b151f350a, entries=150, sequenceid=416, filesize=12.0 K 2024-11-14T09:15:05,746 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~364.17 KB/372912, currentSize=53.67 KB/54960 for 8664731a05c52a198640968a7aa48527 in 858ms, sequenceid=416, compaction requested=true 2024-11-14T09:15:05,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8664731a05c52a198640968a7aa48527: 2024-11-14T09:15:05,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8664731a05c52a198640968a7aa48527:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:15:05,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:15:05,746 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:15:05,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8664731a05c52a198640968a7aa48527:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:15:05,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:15:05,746 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:15:05,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8664731a05c52a198640968a7aa48527:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:15:05,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:15:05,747 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35243 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:15:05,747 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:15:05,747 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): 8664731a05c52a198640968a7aa48527/A is initiating minor compaction (all files) 2024-11-14T09:15:05,747 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): 8664731a05c52a198640968a7aa48527/B is initiating minor compaction (all files) 2024-11-14T09:15:05,747 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8664731a05c52a198640968a7aa48527/A in TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:15:05,747 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8664731a05c52a198640968a7aa48527/B in TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:15:05,747 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/0f839c0d21d0462f9817a0dd1a003b12, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/e9d02b6889554183af684b7115a1e8be, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/34adbd921d6d49fe9f2c07e453be155a] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp, totalSize=36.8 K 2024-11-14T09:15:05,747 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/b9a2d1bc22b74c93a6fc53061a41afbe, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/de2f0479f83f4d7692ea4df2f3c418d4, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/d8ca9864a3bc4f99badfeab27a91862a] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp, totalSize=34.4 K 2024-11-14T09:15:05,748 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f839c0d21d0462f9817a0dd1a003b12, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731575702820 2024-11-14T09:15:05,748 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting b9a2d1bc22b74c93a6fc53061a41afbe, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731575702820 2024-11-14T09:15:05,748 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9d02b6889554183af684b7115a1e8be, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1731575703496 2024-11-14T09:15:05,748 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting de2f0479f83f4d7692ea4df2f3c418d4, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1731575703496 2024-11-14T09:15:05,748 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting d8ca9864a3bc4f99badfeab27a91862a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731575704866 2024-11-14T09:15:05,749 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34adbd921d6d49fe9f2c07e453be155a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731575704866 2024-11-14T09:15:05,772 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8664731a05c52a198640968a7aa48527#B#compaction#283 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:15:05,772 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8664731a05c52a198640968a7aa48527#A#compaction#282 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:15:05,773 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/3871a0add9cf4dcaadd4e43ac173a7d4 is 50, key is test_row_0/A:col10/1731575704881/Put/seqid=0 2024-11-14T09:15:05,773 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/8ba3d3e7a2164dfa8ca92f62679a5a6d is 50, key is test_row_0/B:col10/1731575704881/Put/seqid=0 2024-11-14T09:15:05,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742154_1330 (size=13187) 2024-11-14T09:15:05,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742155_1331 (size=13187) 2024-11-14T09:15:05,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-14T09:15:05,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:15:05,791 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 8664731a05c52a198640968a7aa48527 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-14T09:15:05,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=A 2024-11-14T09:15:05,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:15:05,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=B 2024-11-14T09:15:05,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:15:05,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=C 2024-11-14T09:15:05,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:15:05,796 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/8ba3d3e7a2164dfa8ca92f62679a5a6d as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/8ba3d3e7a2164dfa8ca92f62679a5a6d 2024-11-14T09:15:05,796 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/3871a0add9cf4dcaadd4e43ac173a7d4 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/3871a0add9cf4dcaadd4e43ac173a7d4 2024-11-14T09:15:05,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/40589981d0a544789763fbc3b8287a9e is 50, key is test_row_0/A:col10/1731575704930/Put/seqid=0 2024-11-14T09:15:05,803 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8664731a05c52a198640968a7aa48527/B of 8664731a05c52a198640968a7aa48527 into 8ba3d3e7a2164dfa8ca92f62679a5a6d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:15:05,803 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8664731a05c52a198640968a7aa48527: 2024-11-14T09:15:05,803 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., storeName=8664731a05c52a198640968a7aa48527/B, priority=13, startTime=1731575705746; duration=0sec 2024-11-14T09:15:05,803 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:15:05,803 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8664731a05c52a198640968a7aa48527:B 2024-11-14T09:15:05,804 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:15:05,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-14T09:15:05,809 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35243 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:15:05,809 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): 8664731a05c52a198640968a7aa48527/C is initiating minor compaction (all files) 2024-11-14T09:15:05,809 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8664731a05c52a198640968a7aa48527/C in TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:15:05,809 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/4184d35c74ff4981977979c6e96d9aa6, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/d2602f7f5ed94676afa91f3a1812fbe3, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/56f7a5ad05fc40ce8546db5b151f350a] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp, totalSize=34.4 K 2024-11-14T09:15:05,810 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 4184d35c74ff4981977979c6e96d9aa6, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731575702820 2024-11-14T09:15:05,810 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting d2602f7f5ed94676afa91f3a1812fbe3, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1731575703496 2024-11-14T09:15:05,811 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 56f7a5ad05fc40ce8546db5b151f350a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731575704866 2024-11-14T09:15:05,811 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8664731a05c52a198640968a7aa48527/A of 8664731a05c52a198640968a7aa48527 into 3871a0add9cf4dcaadd4e43ac173a7d4(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:15:05,811 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8664731a05c52a198640968a7aa48527: 2024-11-14T09:15:05,811 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., storeName=8664731a05c52a198640968a7aa48527/A, priority=13, startTime=1731575705746; duration=0sec 2024-11-14T09:15:05,811 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:15:05,811 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8664731a05c52a198640968a7aa48527:A 2024-11-14T09:15:05,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742156_1332 (size=12301) 2024-11-14T09:15:05,816 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/40589981d0a544789763fbc3b8287a9e 2024-11-14T09:15:05,821 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8664731a05c52a198640968a7aa48527#C#compaction#285 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:15:05,821 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/b1bcc501e3ef4c618f97efc446e31583 is 50, key is test_row_0/C:col10/1731575704881/Put/seqid=0 2024-11-14T09:15:05,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/a06a12c4e7aa43ba956aa04a1b820bd9 is 50, key is test_row_0/B:col10/1731575704930/Put/seqid=0 2024-11-14T09:15:05,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742158_1334 (size=12301) 2024-11-14T09:15:05,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742157_1333 (size=13187) 2024-11-14T09:15:05,850 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/b1bcc501e3ef4c618f97efc446e31583 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/b1bcc501e3ef4c618f97efc446e31583 2024-11-14T09:15:05,856 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8664731a05c52a198640968a7aa48527/C of 8664731a05c52a198640968a7aa48527 into b1bcc501e3ef4c618f97efc446e31583(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:15:05,856 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8664731a05c52a198640968a7aa48527: 2024-11-14T09:15:05,856 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., storeName=8664731a05c52a198640968a7aa48527/C, priority=13, startTime=1731575705746; duration=0sec 2024-11-14T09:15:05,856 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:15:05,856 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8664731a05c52a198640968a7aa48527:C 2024-11-14T09:15:06,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on 8664731a05c52a198640968a7aa48527 2024-11-14T09:15:06,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. as already flushing 2024-11-14T09:15:06,161 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731575706136, max timestamp=1731575706157Num uniques 859; 2024-11-14T09:15:06,161 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731575706136, max timestamp=1731575706157Num uniques 859; 2024-11-14T09:15:06,161 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:15:06,161 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731575706136, max timestamp=1731575706157Num uniques 859; 2024-11-14T09:15:06,161 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:15:06,161 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:15:06,236 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/a06a12c4e7aa43ba956aa04a1b820bd9 2024-11-14T09:15:06,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:06,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58122 deadline: 1731575766251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:06,253 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:06,253 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:06,253 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:06,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/37b81ae994924a8a9360aaf42f694166 is 50, key is test_row_0/C:col10/1731575704930/Put/seqid=0 2024-11-14T09:15:06,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742159_1335 (size=12301) 2024-11-14T09:15:06,295 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/37b81ae994924a8a9360aaf42f694166 2024-11-14T09:15:06,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/40589981d0a544789763fbc3b8287a9e as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/40589981d0a544789763fbc3b8287a9e 2024-11-14T09:15:06,307 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/40589981d0a544789763fbc3b8287a9e, entries=150, sequenceid=428, filesize=12.0 K 2024-11-14T09:15:06,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/a06a12c4e7aa43ba956aa04a1b820bd9 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/a06a12c4e7aa43ba956aa04a1b820bd9 2024-11-14T09:15:06,312 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/a06a12c4e7aa43ba956aa04a1b820bd9, entries=150, sequenceid=428, filesize=12.0 K 2024-11-14T09:15:06,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/37b81ae994924a8a9360aaf42f694166 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/37b81ae994924a8a9360aaf42f694166 2024-11-14T09:15:06,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-14T09:15:06,317 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/37b81ae994924a8a9360aaf42f694166, entries=150, sequenceid=428, filesize=12.0 K 2024-11-14T09:15:06,318 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 8664731a05c52a198640968a7aa48527 in 527ms, sequenceid=428, compaction requested=false 2024-11-14T09:15:06,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 8664731a05c52a198640968a7aa48527: 2024-11-14T09:15:06,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:15:06,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-14T09:15:06,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-14T09:15:06,321 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-14T09:15:06,322 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1400 sec 2024-11-14T09:15:06,323 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.1460 sec 2024-11-14T09:15:06,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on 8664731a05c52a198640968a7aa48527 2024-11-14T09:15:06,357 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8664731a05c52a198640968a7aa48527 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-14T09:15:06,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=A 2024-11-14T09:15:06,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:15:06,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=B 2024-11-14T09:15:06,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:15:06,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=C 2024-11-14T09:15:06,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:15:06,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/d03978c984e349c2aadebc15abc1a1e5 is 50, key is test_row_0/A:col10/1731575706235/Put/seqid=0 2024-11-14T09:15:06,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742160_1336 (size=14741) 2024-11-14T09:15:06,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:06,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58122 deadline: 1731575766396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:06,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:06,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:06,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:06,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:06,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58122 deadline: 1731575766505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:06,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:06,508 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:06,508 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:06,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:06,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58122 deadline: 1731575766715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:06,720 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:06,720 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:06,720 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:06,767 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/d03978c984e349c2aadebc15abc1a1e5 2024-11-14T09:15:06,775 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/132f3ecd09074a6ea1cea38e240444ba is 50, key is test_row_0/B:col10/1731575706235/Put/seqid=0 2024-11-14T09:15:06,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742161_1337 (size=12301) 2024-11-14T09:15:06,958 INFO [Thread-1175 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1175. 2024-11-14T09:15:06,958 DEBUG [Thread-1175 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:15:06,958 DEBUG [Thread-1175 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:06,958 DEBUG [Thread-1175 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:06,958 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:15:06,959 INFO [Thread-1177 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1177. 2024-11-14T09:15:06,959 DEBUG [Thread-1177 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:15:06,959 DEBUG [Thread-1177 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:06,960 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:15:06,960 DEBUG [Thread-1177 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:06,960 INFO [Thread-1179 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1179. 2024-11-14T09:15:06,960 DEBUG [Thread-1179 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:15:06,960 DEBUG [Thread-1179 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:06,960 DEBUG [Thread-1179 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:06,960 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:15:06,961 INFO [Thread-1183 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1183. 2024-11-14T09:15:06,962 DEBUG [Thread-1183 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:15:06,962 DEBUG [Thread-1183 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:06,962 DEBUG [Thread-1183 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:06,962 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:15:06,964 INFO [Thread-1181 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1181. 2024-11-14T09:15:06,964 DEBUG [Thread-1181 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:15:06,964 DEBUG [Thread-1181 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:06,964 DEBUG [Thread-1181 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:06,964 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:15:07,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:07,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58122 deadline: 1731575767025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:07,025 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:07,025 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:07,025 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:07,180 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/132f3ecd09074a6ea1cea38e240444ba 2024-11-14T09:15:07,186 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/e83c0c62bc3d4d23bf5d473cba15fc7a is 50, key is test_row_0/C:col10/1731575706235/Put/seqid=0 2024-11-14T09:15:07,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742162_1338 (size=12301) 2024-11-14T09:15:07,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-14T09:15:07,325 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:15:07,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:07,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58122 deadline: 1731575767535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:07,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:07,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:07,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:07,590 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/e83c0c62bc3d4d23bf5d473cba15fc7a 2024-11-14T09:15:07,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/d03978c984e349c2aadebc15abc1a1e5 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/d03978c984e349c2aadebc15abc1a1e5 2024-11-14T09:15:07,597 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/d03978c984e349c2aadebc15abc1a1e5, entries=200, sequenceid=468, filesize=14.4 K 2024-11-14T09:15:07,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/132f3ecd09074a6ea1cea38e240444ba as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/132f3ecd09074a6ea1cea38e240444ba 2024-11-14T09:15:07,600 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/132f3ecd09074a6ea1cea38e240444ba, entries=150, sequenceid=468, filesize=12.0 K 2024-11-14T09:15:07,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/e83c0c62bc3d4d23bf5d473cba15fc7a as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/e83c0c62bc3d4d23bf5d473cba15fc7a 2024-11-14T09:15:07,603 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/e83c0c62bc3d4d23bf5d473cba15fc7a, entries=150, sequenceid=468, filesize=12.0 K 2024-11-14T09:15:07,604 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 8664731a05c52a198640968a7aa48527 in 1247ms, sequenceid=468, compaction requested=true 2024-11-14T09:15:07,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8664731a05c52a198640968a7aa48527: 2024-11-14T09:15:07,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8664731a05c52a198640968a7aa48527:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:15:07,604 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:15:07,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:15:07,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8664731a05c52a198640968a7aa48527:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:15:07,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:15:07,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8664731a05c52a198640968a7aa48527:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:15:07,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:15:07,604 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:15:07,605 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:15:07,605 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:15:07,605 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): 8664731a05c52a198640968a7aa48527/B is initiating minor compaction (all files) 2024-11-14T09:15:07,605 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): 8664731a05c52a198640968a7aa48527/A is initiating minor compaction (all files) 2024-11-14T09:15:07,605 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8664731a05c52a198640968a7aa48527/B in TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:15:07,605 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8664731a05c52a198640968a7aa48527/A in TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:15:07,605 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/8ba3d3e7a2164dfa8ca92f62679a5a6d, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/a06a12c4e7aa43ba956aa04a1b820bd9, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/132f3ecd09074a6ea1cea38e240444ba] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp, totalSize=36.9 K 2024-11-14T09:15:07,605 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/3871a0add9cf4dcaadd4e43ac173a7d4, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/40589981d0a544789763fbc3b8287a9e, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/d03978c984e349c2aadebc15abc1a1e5] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp, totalSize=39.3 K 2024-11-14T09:15:07,606 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ba3d3e7a2164dfa8ca92f62679a5a6d, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731575704866 2024-11-14T09:15:07,606 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3871a0add9cf4dcaadd4e43ac173a7d4, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731575704866 2024-11-14T09:15:07,606 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting a06a12c4e7aa43ba956aa04a1b820bd9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1731575704912 2024-11-14T09:15:07,606 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40589981d0a544789763fbc3b8287a9e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1731575704912 2024-11-14T09:15:07,606 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 132f3ecd09074a6ea1cea38e240444ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1731575706224 2024-11-14T09:15:07,606 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting d03978c984e349c2aadebc15abc1a1e5, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1731575706224 2024-11-14T09:15:07,613 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8664731a05c52a198640968a7aa48527#B#compaction#292 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:15:07,613 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8664731a05c52a198640968a7aa48527#A#compaction#291 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:15:07,613 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/3daa4659624645ce86a7e5a1b84ba635 is 50, key is test_row_0/B:col10/1731575706235/Put/seqid=0 2024-11-14T09:15:07,613 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/37fd354760994ffd9d517de08e1c369c is 50, key is test_row_0/A:col10/1731575706235/Put/seqid=0 2024-11-14T09:15:07,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742163_1339 (size=13289) 2024-11-14T09:15:07,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742164_1340 (size=13289) 2024-11-14T09:15:08,024 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/37fd354760994ffd9d517de08e1c369c as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/37fd354760994ffd9d517de08e1c369c 2024-11-14T09:15:08,025 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/3daa4659624645ce86a7e5a1b84ba635 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/3daa4659624645ce86a7e5a1b84ba635 2024-11-14T09:15:08,029 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8664731a05c52a198640968a7aa48527/A of 8664731a05c52a198640968a7aa48527 into 37fd354760994ffd9d517de08e1c369c(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:15:08,029 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8664731a05c52a198640968a7aa48527: 2024-11-14T09:15:08,029 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., storeName=8664731a05c52a198640968a7aa48527/A, priority=13, startTime=1731575707604; duration=0sec 2024-11-14T09:15:08,029 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8664731a05c52a198640968a7aa48527/B of 8664731a05c52a198640968a7aa48527 into 3daa4659624645ce86a7e5a1b84ba635(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:15:08,029 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8664731a05c52a198640968a7aa48527: 2024-11-14T09:15:08,029 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., storeName=8664731a05c52a198640968a7aa48527/B, priority=13, startTime=1731575707604; duration=0sec 2024-11-14T09:15:08,029 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:15:08,029 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:15:08,029 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8664731a05c52a198640968a7aa48527:A 2024-11-14T09:15:08,029 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8664731a05c52a198640968a7aa48527:B 2024-11-14T09:15:08,029 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:15:08,030 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:15:08,030 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): 8664731a05c52a198640968a7aa48527/C is initiating minor compaction (all files) 2024-11-14T09:15:08,030 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8664731a05c52a198640968a7aa48527/C in TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:15:08,030 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/b1bcc501e3ef4c618f97efc446e31583, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/37b81ae994924a8a9360aaf42f694166, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/e83c0c62bc3d4d23bf5d473cba15fc7a] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp, totalSize=36.9 K 2024-11-14T09:15:08,030 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1bcc501e3ef4c618f97efc446e31583, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731575704866 2024-11-14T09:15:08,030 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37b81ae994924a8a9360aaf42f694166, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1731575704912 2024-11-14T09:15:08,031 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting e83c0c62bc3d4d23bf5d473cba15fc7a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1731575706224 2024-11-14T09:15:08,036 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8664731a05c52a198640968a7aa48527#C#compaction#293 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:15:08,037 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/3472edff169340928b3e757531bedee4 is 50, key is test_row_0/C:col10/1731575706235/Put/seqid=0 2024-11-14T09:15:08,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742165_1341 (size=13289) 2024-11-14T09:15:08,445 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/3472edff169340928b3e757531bedee4 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/3472edff169340928b3e757531bedee4 2024-11-14T09:15:08,449 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8664731a05c52a198640968a7aa48527/C of 8664731a05c52a198640968a7aa48527 into 3472edff169340928b3e757531bedee4(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:15:08,449 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8664731a05c52a198640968a7aa48527: 2024-11-14T09:15:08,449 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., storeName=8664731a05c52a198640968a7aa48527/C, priority=13, startTime=1731575707604; duration=0sec 2024-11-14T09:15:08,449 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:15:08,449 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8664731a05c52a198640968a7aa48527:C 2024-11-14T09:15:08,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on 8664731a05c52a198640968a7aa48527 2024-11-14T09:15:08,546 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8664731a05c52a198640968a7aa48527 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-14T09:15:08,546 INFO [Thread-1166 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1166. 2024-11-14T09:15:08,547 DEBUG [Thread-1166 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:15:08,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=A 2024-11-14T09:15:08,547 DEBUG [Thread-1166 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:08,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:15:08,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=B 2024-11-14T09:15:08,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:15:08,547 DEBUG [Thread-1166 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:08,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=C 2024-11-14T09:15:08,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:15:08,547 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:15:08,550 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/ce928c1160d84020a6d772ec1ebb1e22 is 50, key is test_row_0/A:col10/1731575706394/Put/seqid=0 2024-11-14T09:15:08,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742166_1342 (size=12301) 2024-11-14T09:15:08,697 INFO [Thread-1170 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1170. 2024-11-14T09:15:08,697 DEBUG [Thread-1170 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:15:08,697 DEBUG [Thread-1170 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:08,697 DEBUG [Thread-1170 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:08,697 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:15:08,756 INFO [Thread-1168 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1168. 2024-11-14T09:15:08,756 DEBUG [Thread-1168 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:15:08,756 DEBUG [Thread-1168 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:08,756 DEBUG [Thread-1168 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:08,756 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:15:08,954 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=482 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/ce928c1160d84020a6d772ec1ebb1e22 2024-11-14T09:15:08,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/aadb82f7cc1d4d6e8d28dcd2abb2b846 is 50, key is test_row_0/B:col10/1731575706394/Put/seqid=0 2024-11-14T09:15:08,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742167_1343 (size=12301) 2024-11-14T09:15:09,117 INFO [Thread-1164 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1164. 2024-11-14T09:15:09,117 DEBUG [Thread-1164 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:15:09,117 DEBUG [Thread-1164 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:09,117 DEBUG [Thread-1164 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:09,117 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:15:09,364 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=482 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/aadb82f7cc1d4d6e8d28dcd2abb2b846 2024-11-14T09:15:09,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/f8e257b2bb7449ec943d0642e53f57db is 50, key is test_row_0/C:col10/1731575706394/Put/seqid=0 2024-11-14T09:15:09,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742168_1344 (size=12301) 2024-11-14T09:15:09,774 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=482 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/f8e257b2bb7449ec943d0642e53f57db 2024-11-14T09:15:09,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/ce928c1160d84020a6d772ec1ebb1e22 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/ce928c1160d84020a6d772ec1ebb1e22 2024-11-14T09:15:09,781 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/ce928c1160d84020a6d772ec1ebb1e22, entries=150, sequenceid=482, filesize=12.0 K 2024-11-14T09:15:09,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/aadb82f7cc1d4d6e8d28dcd2abb2b846 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/aadb82f7cc1d4d6e8d28dcd2abb2b846 2024-11-14T09:15:09,784 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/aadb82f7cc1d4d6e8d28dcd2abb2b846, entries=150, sequenceid=482, filesize=12.0 K 2024-11-14T09:15:09,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/f8e257b2bb7449ec943d0642e53f57db as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/f8e257b2bb7449ec943d0642e53f57db 2024-11-14T09:15:09,788 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/f8e257b2bb7449ec943d0642e53f57db, entries=150, sequenceid=482, filesize=12.0 K 2024-11-14T09:15:09,789 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=20.13 KB/20610 for 8664731a05c52a198640968a7aa48527 in 1243ms, sequenceid=482, compaction requested=false 2024-11-14T09:15:09,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8664731a05c52a198640968a7aa48527: 2024-11-14T09:15:10,753 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T09:15:15,796 INFO [Thread-1172 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1172. 2024-11-14T09:15:15,796 DEBUG [Thread-1172 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:15:15,796 DEBUG [Thread-1172 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:15,796 DEBUG [Thread-1172 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:15,796 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:15:15,797 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T09:15:15,797 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-11-14T09:15:15,797 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 182 2024-11-14T09:15:15,797 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 22 2024-11-14T09:15:15,797 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 25 2024-11-14T09:15:15,797 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 112 2024-11-14T09:15:15,797 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T09:15:15,797 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3342 2024-11-14T09:15:15,797 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3223 2024-11-14T09:15:15,797 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3057 2024-11-14T09:15:15,797 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3338 2024-11-14T09:15:15,797 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3204 2024-11-14T09:15:15,797 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T09:15:15,797 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T09:15:15,797 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T09:15:15,797 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:15,797 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:15,797 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:15:15,798 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-14T09:15:15,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T09:15:15,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-14T09:15:15,800 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731575715800"}]},"ts":"1731575715800"} 2024-11-14T09:15:15,802 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T09:15:15,802 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T09:15:15,802 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T09:15:15,803 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8664731a05c52a198640968a7aa48527, UNASSIGN}] 2024-11-14T09:15:15,804 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8664731a05c52a198640968a7aa48527, UNASSIGN 2024-11-14T09:15:15,804 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=8664731a05c52a198640968a7aa48527, regionState=CLOSING, regionLocation=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:15,806 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8664731a05c52a198640968a7aa48527, UNASSIGN because future has completed 2024-11-14T09:15:15,806 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T09:15:15,806 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393}] 2024-11-14T09:15:15,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-14T09:15:15,958 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(122): Close 8664731a05c52a198640968a7aa48527 2024-11-14T09:15:15,958 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T09:15:15,958 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1722): Closing 8664731a05c52a198640968a7aa48527, disabling compactions & flushes 2024-11-14T09:15:15,958 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:15:15,958 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:15:15,958 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. after waiting 0 ms 2024-11-14T09:15:15,958 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:15:15,958 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(2902): Flushing 8664731a05c52a198640968a7aa48527 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-14T09:15:15,958 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=A 2024-11-14T09:15:15,958 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:15:15,958 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=B 2024-11-14T09:15:15,958 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:15:15,958 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8664731a05c52a198640968a7aa48527, store=C 2024-11-14T09:15:15,958 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:15:15,962 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/760be72603d64bac95bac1eb19a732b7 is 50, key is test_row_1/A:col10/1731575709116/Put/seqid=0 2024-11-14T09:15:15,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742169_1345 (size=9857) 2024-11-14T09:15:16,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-14T09:15:16,365 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/760be72603d64bac95bac1eb19a732b7 2024-11-14T09:15:16,372 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/c9b9e90a46124dddaa37ce0f30f86f62 is 50, key is test_row_1/B:col10/1731575709116/Put/seqid=0 2024-11-14T09:15:16,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742170_1346 (size=9857) 2024-11-14T09:15:16,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-14T09:15:16,776 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/c9b9e90a46124dddaa37ce0f30f86f62 2024-11-14T09:15:16,781 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/197ba8dea75d4ed19d09f8993be6e821 is 50, key is test_row_1/C:col10/1731575709116/Put/seqid=0 2024-11-14T09:15:16,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742171_1347 (size=9857) 2024-11-14T09:15:16,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-14T09:15:17,185 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/197ba8dea75d4ed19d09f8993be6e821 2024-11-14T09:15:17,189 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/A/760be72603d64bac95bac1eb19a732b7 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/760be72603d64bac95bac1eb19a732b7 2024-11-14T09:15:17,192 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/760be72603d64bac95bac1eb19a732b7, entries=100, sequenceid=489, filesize=9.6 K 2024-11-14T09:15:17,193 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/B/c9b9e90a46124dddaa37ce0f30f86f62 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/c9b9e90a46124dddaa37ce0f30f86f62 2024-11-14T09:15:17,195 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/c9b9e90a46124dddaa37ce0f30f86f62, entries=100, sequenceid=489, filesize=9.6 K 2024-11-14T09:15:17,196 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/.tmp/C/197ba8dea75d4ed19d09f8993be6e821 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/197ba8dea75d4ed19d09f8993be6e821 2024-11-14T09:15:17,199 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/197ba8dea75d4ed19d09f8993be6e821, entries=100, sequenceid=489, filesize=9.6 K 2024-11-14T09:15:17,199 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 8664731a05c52a198640968a7aa48527 in 1241ms, sequenceid=489, compaction requested=true 2024-11-14T09:15:17,200 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/d13de21550fd464caced9b80ccc430e3, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/4792663d6d1246f880d9ddabff255221, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/efb9fa9402e741588dc5f0a8cacd4c1d, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/e41821b919cf41bcb5757ec0e52eaf8d, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/aa359d9fcbf54b9ea0aa546cf88c653e, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/4892896493724401aa747e870dfe2be2, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/2df63f3e815143f5acd4e65b88ca88ec, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/c62875c5ccc14c9a8d6b781c248922b8, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/2442a1b87d4446ef846301da5a9e8394, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/5afc0de76abf4af187352ddb72418307, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/074f96b812d046c792a28eabc64d317f, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/ddc04f5f4c5e4b509df609f3d3e1b6d2, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/95e26fc09aa34f31af344e6f909b47ec, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/1e73dbfc059b4b869b7a48c7ed25632e, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/df844f87055140409c43490383c5676c, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/3b1cfcc863f4439d8c1dcc514fcce097, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/4e6de91173f0494da9c9be9b5fea1678, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/ba6b6324f13842f1931bd781fc8c725d, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/c4d4af63ca9e4568a0024d498871c06f, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/ca30e32004d346a8b030c0428c3ae5f4, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/824215931234457db1a42e4a04433c7e, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/f6e6948eca374994b7419eb312eb974a, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/fc08047c6b63468d9de846d9d1400d41, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/0f839c0d21d0462f9817a0dd1a003b12, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/e9d02b6889554183af684b7115a1e8be, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/34adbd921d6d49fe9f2c07e453be155a, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/3871a0add9cf4dcaadd4e43ac173a7d4, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/40589981d0a544789763fbc3b8287a9e, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/d03978c984e349c2aadebc15abc1a1e5] to archive 2024-11-14T09:15:17,201 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T09:15:17,202 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/d13de21550fd464caced9b80ccc430e3 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/d13de21550fd464caced9b80ccc430e3 2024-11-14T09:15:17,203 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/4792663d6d1246f880d9ddabff255221 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/4792663d6d1246f880d9ddabff255221 2024-11-14T09:15:17,204 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/efb9fa9402e741588dc5f0a8cacd4c1d to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/efb9fa9402e741588dc5f0a8cacd4c1d 2024-11-14T09:15:17,205 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/e41821b919cf41bcb5757ec0e52eaf8d to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/e41821b919cf41bcb5757ec0e52eaf8d 2024-11-14T09:15:17,205 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/aa359d9fcbf54b9ea0aa546cf88c653e to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/aa359d9fcbf54b9ea0aa546cf88c653e 2024-11-14T09:15:17,206 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/4892896493724401aa747e870dfe2be2 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/4892896493724401aa747e870dfe2be2 2024-11-14T09:15:17,207 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/2df63f3e815143f5acd4e65b88ca88ec to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/2df63f3e815143f5acd4e65b88ca88ec 2024-11-14T09:15:17,208 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/c62875c5ccc14c9a8d6b781c248922b8 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/c62875c5ccc14c9a8d6b781c248922b8 2024-11-14T09:15:17,209 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/2442a1b87d4446ef846301da5a9e8394 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/2442a1b87d4446ef846301da5a9e8394 2024-11-14T09:15:17,210 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/5afc0de76abf4af187352ddb72418307 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/5afc0de76abf4af187352ddb72418307 2024-11-14T09:15:17,211 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/074f96b812d046c792a28eabc64d317f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/074f96b812d046c792a28eabc64d317f 2024-11-14T09:15:17,212 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/ddc04f5f4c5e4b509df609f3d3e1b6d2 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/ddc04f5f4c5e4b509df609f3d3e1b6d2 2024-11-14T09:15:17,213 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/95e26fc09aa34f31af344e6f909b47ec to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/95e26fc09aa34f31af344e6f909b47ec 2024-11-14T09:15:17,214 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/1e73dbfc059b4b869b7a48c7ed25632e to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/1e73dbfc059b4b869b7a48c7ed25632e 2024-11-14T09:15:17,215 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/df844f87055140409c43490383c5676c to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/df844f87055140409c43490383c5676c 2024-11-14T09:15:17,216 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/3b1cfcc863f4439d8c1dcc514fcce097 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/3b1cfcc863f4439d8c1dcc514fcce097 2024-11-14T09:15:17,216 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/4e6de91173f0494da9c9be9b5fea1678 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/4e6de91173f0494da9c9be9b5fea1678 2024-11-14T09:15:17,217 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/ba6b6324f13842f1931bd781fc8c725d to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/ba6b6324f13842f1931bd781fc8c725d 2024-11-14T09:15:17,218 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/c4d4af63ca9e4568a0024d498871c06f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/c4d4af63ca9e4568a0024d498871c06f 2024-11-14T09:15:17,219 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/ca30e32004d346a8b030c0428c3ae5f4 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/ca30e32004d346a8b030c0428c3ae5f4 2024-11-14T09:15:17,220 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/824215931234457db1a42e4a04433c7e to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/824215931234457db1a42e4a04433c7e 2024-11-14T09:15:17,221 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/f6e6948eca374994b7419eb312eb974a to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/f6e6948eca374994b7419eb312eb974a 2024-11-14T09:15:17,222 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/fc08047c6b63468d9de846d9d1400d41 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/fc08047c6b63468d9de846d9d1400d41 2024-11-14T09:15:17,223 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/0f839c0d21d0462f9817a0dd1a003b12 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/0f839c0d21d0462f9817a0dd1a003b12 2024-11-14T09:15:17,224 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/e9d02b6889554183af684b7115a1e8be to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/e9d02b6889554183af684b7115a1e8be 2024-11-14T09:15:17,225 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/34adbd921d6d49fe9f2c07e453be155a to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/34adbd921d6d49fe9f2c07e453be155a 2024-11-14T09:15:17,226 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/3871a0add9cf4dcaadd4e43ac173a7d4 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/3871a0add9cf4dcaadd4e43ac173a7d4 2024-11-14T09:15:17,227 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/40589981d0a544789763fbc3b8287a9e to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/40589981d0a544789763fbc3b8287a9e 2024-11-14T09:15:17,228 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/d03978c984e349c2aadebc15abc1a1e5 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/d03978c984e349c2aadebc15abc1a1e5 2024-11-14T09:15:17,229 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/a40146e7efc94717bb927bf0f3699bfb, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/ad323a1546aa4d969ae3dd3b5b7dbf2a, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/4daad9fc31564870ba30f63a88de12a5, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/e0104697d5404e9d94ad3b0f1158c489, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/eb8ae91782014f808a9d4c8efac1e7cb, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/af396144976f420d857994aa08f861a5, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/f9d5f3ced1874fb38095f90db028a8dc, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/b1f84d82d7984ea7ba859b76c1504abc, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/8b0904923b094cb2b03d6c8ecf9a1edc, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/a833de38560a4db28dd567434c3dee08, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/bd70e73d6438497d872fc2ffeed2f44c, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/4263f94563a04cc991f0a4677b890e20, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/72a6189ff6c74bb1bb1df1a6fe707178, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/5f702a1007c24349af6f5971ba2e2cc0, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/a059c52f391d4c2b89c756d040f1bf46, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/b8fb581b68d54d6789ed86067acb29af, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/95afab003c5e46b8949968d115ef2052, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/e970e565340a4b5cb9a43ff294869ecd, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/183c8239357d4beda4e2a270eaa93410, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/0529993b8b5b411797507d94240f6d21, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/f9465e73e68c4d1bb8953155fbd06d2c, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/d4dc39b5ee6942e6bc43413d7dcd63a1, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/b9a2d1bc22b74c93a6fc53061a41afbe, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/1f6d7edb8253458b9137776c4462ed0b, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/de2f0479f83f4d7692ea4df2f3c418d4, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/8ba3d3e7a2164dfa8ca92f62679a5a6d, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/d8ca9864a3bc4f99badfeab27a91862a, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/a06a12c4e7aa43ba956aa04a1b820bd9, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/132f3ecd09074a6ea1cea38e240444ba] to archive 2024-11-14T09:15:17,230 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T09:15:17,231 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/a40146e7efc94717bb927bf0f3699bfb to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/a40146e7efc94717bb927bf0f3699bfb 2024-11-14T09:15:17,232 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/ad323a1546aa4d969ae3dd3b5b7dbf2a to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/ad323a1546aa4d969ae3dd3b5b7dbf2a 2024-11-14T09:15:17,233 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/4daad9fc31564870ba30f63a88de12a5 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/4daad9fc31564870ba30f63a88de12a5 2024-11-14T09:15:17,234 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/e0104697d5404e9d94ad3b0f1158c489 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/e0104697d5404e9d94ad3b0f1158c489 2024-11-14T09:15:17,235 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/eb8ae91782014f808a9d4c8efac1e7cb to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/eb8ae91782014f808a9d4c8efac1e7cb 2024-11-14T09:15:17,236 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/af396144976f420d857994aa08f861a5 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/af396144976f420d857994aa08f861a5 2024-11-14T09:15:17,236 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/f9d5f3ced1874fb38095f90db028a8dc to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/f9d5f3ced1874fb38095f90db028a8dc 2024-11-14T09:15:17,237 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/b1f84d82d7984ea7ba859b76c1504abc to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/b1f84d82d7984ea7ba859b76c1504abc 2024-11-14T09:15:17,238 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/8b0904923b094cb2b03d6c8ecf9a1edc to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/8b0904923b094cb2b03d6c8ecf9a1edc 2024-11-14T09:15:17,239 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/a833de38560a4db28dd567434c3dee08 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/a833de38560a4db28dd567434c3dee08 2024-11-14T09:15:17,240 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/bd70e73d6438497d872fc2ffeed2f44c to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/bd70e73d6438497d872fc2ffeed2f44c 2024-11-14T09:15:17,241 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/4263f94563a04cc991f0a4677b890e20 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/4263f94563a04cc991f0a4677b890e20 2024-11-14T09:15:17,241 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/72a6189ff6c74bb1bb1df1a6fe707178 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/72a6189ff6c74bb1bb1df1a6fe707178 2024-11-14T09:15:17,242 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/5f702a1007c24349af6f5971ba2e2cc0 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/5f702a1007c24349af6f5971ba2e2cc0 2024-11-14T09:15:17,243 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/a059c52f391d4c2b89c756d040f1bf46 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/a059c52f391d4c2b89c756d040f1bf46 2024-11-14T09:15:17,244 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/b8fb581b68d54d6789ed86067acb29af to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/b8fb581b68d54d6789ed86067acb29af 2024-11-14T09:15:17,245 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/95afab003c5e46b8949968d115ef2052 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/95afab003c5e46b8949968d115ef2052 2024-11-14T09:15:17,246 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/e970e565340a4b5cb9a43ff294869ecd to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/e970e565340a4b5cb9a43ff294869ecd 2024-11-14T09:15:17,246 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/183c8239357d4beda4e2a270eaa93410 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/183c8239357d4beda4e2a270eaa93410 2024-11-14T09:15:17,247 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/0529993b8b5b411797507d94240f6d21 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/0529993b8b5b411797507d94240f6d21 2024-11-14T09:15:17,248 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/f9465e73e68c4d1bb8953155fbd06d2c to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/f9465e73e68c4d1bb8953155fbd06d2c 2024-11-14T09:15:17,249 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/d4dc39b5ee6942e6bc43413d7dcd63a1 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/d4dc39b5ee6942e6bc43413d7dcd63a1 2024-11-14T09:15:17,250 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/b9a2d1bc22b74c93a6fc53061a41afbe to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/b9a2d1bc22b74c93a6fc53061a41afbe 2024-11-14T09:15:17,251 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/1f6d7edb8253458b9137776c4462ed0b to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/1f6d7edb8253458b9137776c4462ed0b 2024-11-14T09:15:17,252 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/de2f0479f83f4d7692ea4df2f3c418d4 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/de2f0479f83f4d7692ea4df2f3c418d4 2024-11-14T09:15:17,253 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/8ba3d3e7a2164dfa8ca92f62679a5a6d to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/8ba3d3e7a2164dfa8ca92f62679a5a6d 2024-11-14T09:15:17,254 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/d8ca9864a3bc4f99badfeab27a91862a to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/d8ca9864a3bc4f99badfeab27a91862a 2024-11-14T09:15:17,254 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/a06a12c4e7aa43ba956aa04a1b820bd9 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/a06a12c4e7aa43ba956aa04a1b820bd9 2024-11-14T09:15:17,255 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/132f3ecd09074a6ea1cea38e240444ba to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/132f3ecd09074a6ea1cea38e240444ba 2024-11-14T09:15:17,257 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/de0b291128bf4494bb95db285538a65e, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/6528463614fd46b48a015541900dac89, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/c6d0aac053af4eabbab58711b99753e6, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/2f0a82a8d0cb42f2bc81ac403b989b71, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/2cff30a1149e4424b52e3b50c527fa94, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/a63b0c8362ea424da1edae51e3efccf7, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/9aa697c613e346fbb15607fca0f20744, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/0835a71643f944cea5b3c875ea880c16, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/12135aabc6c44912a7a7b3ef8c08f1c5, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/2c4a181e23be4766872696eac98a8839, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/c8017cc3c6c5448c8c5b7cce91811bcd, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/05ed99bbd20242edade72be64e7d6da7, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/e245f91696334f6fac42c55910e4fdcb, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/ce32dbd84beb49c2abb7b3e66b6f5e06, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/dc22d4eaa5114ef1986204aaaf7df1ea, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/ae1ea5bbf77d4d8db014f3b642d13310, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/26534f0444824557b8192887022d3274, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/37fd770cd174471185df0fbc8d974fcf, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/f402d66f9ba342f1ab9379d97aa578d2, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/e3c8beb30fb94fa1877c3e3fd686334b, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/c242630cdfb64e129db1c994375549da, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/6b7684187d8444738146c5bed424fe69, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/4184d35c74ff4981977979c6e96d9aa6, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/1eab68bb11134d14a4c09ba323b60440, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/d2602f7f5ed94676afa91f3a1812fbe3, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/b1bcc501e3ef4c618f97efc446e31583, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/56f7a5ad05fc40ce8546db5b151f350a, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/37b81ae994924a8a9360aaf42f694166, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/e83c0c62bc3d4d23bf5d473cba15fc7a] to archive 2024-11-14T09:15:17,257 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T09:15:17,258 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/de0b291128bf4494bb95db285538a65e to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/de0b291128bf4494bb95db285538a65e 2024-11-14T09:15:17,259 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/6528463614fd46b48a015541900dac89 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/6528463614fd46b48a015541900dac89 2024-11-14T09:15:17,260 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/c6d0aac053af4eabbab58711b99753e6 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/c6d0aac053af4eabbab58711b99753e6 2024-11-14T09:15:17,261 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/2f0a82a8d0cb42f2bc81ac403b989b71 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/2f0a82a8d0cb42f2bc81ac403b989b71 2024-11-14T09:15:17,262 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/2cff30a1149e4424b52e3b50c527fa94 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/2cff30a1149e4424b52e3b50c527fa94 2024-11-14T09:15:17,263 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/a63b0c8362ea424da1edae51e3efccf7 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/a63b0c8362ea424da1edae51e3efccf7 2024-11-14T09:15:17,264 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/9aa697c613e346fbb15607fca0f20744 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/9aa697c613e346fbb15607fca0f20744 2024-11-14T09:15:17,265 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/0835a71643f944cea5b3c875ea880c16 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/0835a71643f944cea5b3c875ea880c16 2024-11-14T09:15:17,266 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/12135aabc6c44912a7a7b3ef8c08f1c5 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/12135aabc6c44912a7a7b3ef8c08f1c5 2024-11-14T09:15:17,267 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/2c4a181e23be4766872696eac98a8839 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/2c4a181e23be4766872696eac98a8839 2024-11-14T09:15:17,268 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/c8017cc3c6c5448c8c5b7cce91811bcd to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/c8017cc3c6c5448c8c5b7cce91811bcd 2024-11-14T09:15:17,269 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/05ed99bbd20242edade72be64e7d6da7 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/05ed99bbd20242edade72be64e7d6da7 2024-11-14T09:15:17,270 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/e245f91696334f6fac42c55910e4fdcb to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/e245f91696334f6fac42c55910e4fdcb 2024-11-14T09:15:17,271 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/ce32dbd84beb49c2abb7b3e66b6f5e06 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/ce32dbd84beb49c2abb7b3e66b6f5e06 2024-11-14T09:15:17,272 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/dc22d4eaa5114ef1986204aaaf7df1ea to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/dc22d4eaa5114ef1986204aaaf7df1ea 2024-11-14T09:15:17,273 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/ae1ea5bbf77d4d8db014f3b642d13310 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/ae1ea5bbf77d4d8db014f3b642d13310 2024-11-14T09:15:17,274 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/26534f0444824557b8192887022d3274 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/26534f0444824557b8192887022d3274 2024-11-14T09:15:17,275 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/37fd770cd174471185df0fbc8d974fcf to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/37fd770cd174471185df0fbc8d974fcf 2024-11-14T09:15:17,276 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/f402d66f9ba342f1ab9379d97aa578d2 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/f402d66f9ba342f1ab9379d97aa578d2 2024-11-14T09:15:17,277 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/e3c8beb30fb94fa1877c3e3fd686334b to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/e3c8beb30fb94fa1877c3e3fd686334b 2024-11-14T09:15:17,279 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/c242630cdfb64e129db1c994375549da to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/c242630cdfb64e129db1c994375549da 2024-11-14T09:15:17,279 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/6b7684187d8444738146c5bed424fe69 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/6b7684187d8444738146c5bed424fe69 2024-11-14T09:15:17,280 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/4184d35c74ff4981977979c6e96d9aa6 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/4184d35c74ff4981977979c6e96d9aa6 2024-11-14T09:15:17,281 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/1eab68bb11134d14a4c09ba323b60440 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/1eab68bb11134d14a4c09ba323b60440 2024-11-14T09:15:17,282 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/d2602f7f5ed94676afa91f3a1812fbe3 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/d2602f7f5ed94676afa91f3a1812fbe3 2024-11-14T09:15:17,283 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/b1bcc501e3ef4c618f97efc446e31583 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/b1bcc501e3ef4c618f97efc446e31583 2024-11-14T09:15:17,284 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/56f7a5ad05fc40ce8546db5b151f350a to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/56f7a5ad05fc40ce8546db5b151f350a 2024-11-14T09:15:17,285 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/37b81ae994924a8a9360aaf42f694166 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/37b81ae994924a8a9360aaf42f694166 2024-11-14T09:15:17,286 DEBUG [StoreCloser-TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/e83c0c62bc3d4d23bf5d473cba15fc7a to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/e83c0c62bc3d4d23bf5d473cba15fc7a 2024-11-14T09:15:17,291 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/recovered.edits/492.seqid, newMaxSeqId=492, maxSeqId=1 2024-11-14T09:15:17,291 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. 2024-11-14T09:15:17,291 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1676): Region close journal for 8664731a05c52a198640968a7aa48527: Waiting for close lock at 1731575715958Running coprocessor pre-close hooks at 1731575715958Disabling compacts and flushes for region at 1731575715958Disabling writes for close at 1731575715958Obtaining lock to block concurrent updates at 1731575715958Preparing flush snapshotting stores in 8664731a05c52a198640968a7aa48527 at 1731575715958Finished memstore snapshotting TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731575715958Flushing stores of TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527. at 1731575715959 (+1 ms)Flushing 8664731a05c52a198640968a7aa48527/A: creating writer at 1731575715959Flushing 8664731a05c52a198640968a7aa48527/A: appending metadata at 1731575715961 (+2 ms)Flushing 8664731a05c52a198640968a7aa48527/A: closing flushed file at 1731575715961Flushing 8664731a05c52a198640968a7aa48527/B: creating writer at 1731575716369 (+408 ms)Flushing 8664731a05c52a198640968a7aa48527/B: appending metadata at 1731575716371 (+2 ms)Flushing 8664731a05c52a198640968a7aa48527/B: closing flushed file at 1731575716371Flushing 8664731a05c52a198640968a7aa48527/C: creating writer at 1731575716779 (+408 ms)Flushing 8664731a05c52a198640968a7aa48527/C: appending metadata at 1731575716781 (+2 ms)Flushing 8664731a05c52a198640968a7aa48527/C: closing flushed file at 1731575716781Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3fd05509: reopening flushed file at 1731575717189 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b34bc48: reopening flushed file at 1731575717192 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@70e6a0c0: reopening flushed file at 1731575717195 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 8664731a05c52a198640968a7aa48527 in 1241ms, sequenceid=489, compaction requested=true at 1731575717199 (+4 ms)Writing region close event to WAL at 1731575717288 (+89 ms)Running coprocessor post-close hooks at 1731575717291 (+3 ms)Closed at 1731575717291 2024-11-14T09:15:17,293 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(157): Closed 8664731a05c52a198640968a7aa48527 2024-11-14T09:15:17,293 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=8664731a05c52a198640968a7aa48527, regionState=CLOSED 2024-11-14T09:15:17,295 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 because future has completed 2024-11-14T09:15:17,297 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-14T09:15:17,297 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseRegionProcedure 8664731a05c52a198640968a7aa48527, server=83f56b55f2af,39927,1731575622393 in 1.4890 sec 2024-11-14T09:15:17,298 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-11-14T09:15:17,298 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8664731a05c52a198640968a7aa48527, UNASSIGN in 1.4940 sec 2024-11-14T09:15:17,300 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-14T09:15:17,300 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4970 sec 2024-11-14T09:15:17,301 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731575717301"}]},"ts":"1731575717301"} 2024-11-14T09:15:17,302 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T09:15:17,302 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T09:15:17,303 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5040 sec 2024-11-14T09:15:17,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-14T09:15:17,945 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:15:17,946 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-14T09:15:17,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:15:17,947 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:15:17,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-14T09:15:17,948 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=86, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:15:17,950 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527 2024-11-14T09:15:17,953 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A, FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B, FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C, FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/recovered.edits] 2024-11-14T09:15:17,956 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/37fd354760994ffd9d517de08e1c369c to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/37fd354760994ffd9d517de08e1c369c 2024-11-14T09:15:17,957 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/760be72603d64bac95bac1eb19a732b7 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/760be72603d64bac95bac1eb19a732b7 2024-11-14T09:15:17,958 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/ce928c1160d84020a6d772ec1ebb1e22 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/A/ce928c1160d84020a6d772ec1ebb1e22 2024-11-14T09:15:17,960 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/3daa4659624645ce86a7e5a1b84ba635 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/3daa4659624645ce86a7e5a1b84ba635 2024-11-14T09:15:17,962 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/aadb82f7cc1d4d6e8d28dcd2abb2b846 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/aadb82f7cc1d4d6e8d28dcd2abb2b846 2024-11-14T09:15:17,963 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/c9b9e90a46124dddaa37ce0f30f86f62 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/B/c9b9e90a46124dddaa37ce0f30f86f62 2024-11-14T09:15:17,965 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/197ba8dea75d4ed19d09f8993be6e821 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/197ba8dea75d4ed19d09f8993be6e821 2024-11-14T09:15:17,966 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/3472edff169340928b3e757531bedee4 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/3472edff169340928b3e757531bedee4 2024-11-14T09:15:17,967 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/f8e257b2bb7449ec943d0642e53f57db to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/C/f8e257b2bb7449ec943d0642e53f57db 2024-11-14T09:15:17,970 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/recovered.edits/492.seqid to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527/recovered.edits/492.seqid 2024-11-14T09:15:17,971 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/8664731a05c52a198640968a7aa48527 2024-11-14T09:15:17,971 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T09:15:17,973 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=86, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:15:17,976 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T09:15:17,979 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T09:15:17,980 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=86, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:15:17,980 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T09:15:17,980 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731575717980"}]},"ts":"9223372036854775807"} 2024-11-14T09:15:17,982 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T09:15:17,982 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 8664731a05c52a198640968a7aa48527, NAME => 'TestAcidGuarantees,,1731575684754.8664731a05c52a198640968a7aa48527.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T09:15:17,983 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T09:15:17,983 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731575717983"}]},"ts":"9223372036854775807"} 2024-11-14T09:15:17,984 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T09:15:17,985 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=86, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:15:17,986 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 39 msec 2024-11-14T09:15:18,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-14T09:15:18,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T09:15:18,056 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:15:18,068 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=246 (was 248), OpenFileDescriptor=447 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=450 (was 475), ProcessCount=11 (was 11), AvailableMemoryMB=6788 (was 7422) 2024-11-14T09:15:18,078 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=246, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=450, ProcessCount=11, AvailableMemoryMB=6787 2024-11-14T09:15:18,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T09:15:18,080 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T09:15:18,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T09:15:18,082 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T09:15:18,083 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:15:18,083 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 87 2024-11-14T09:15:18,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-14T09:15:18,083 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T09:15:18,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742172_1348 (size=1061) 2024-11-14T09:15:18,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-14T09:15:18,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-14T09:15:18,492 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db 2024-11-14T09:15:18,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742173_1349 (size=53) 2024-11-14T09:15:18,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-14T09:15:18,897 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:15:18,898 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 3e7b42f1efdfdf4879c3f173e59339f9, disabling compactions & flushes 2024-11-14T09:15:18,898 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:18,898 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:18,898 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. after waiting 0 ms 2024-11-14T09:15:18,898 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:18,898 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:18,898 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3e7b42f1efdfdf4879c3f173e59339f9: Waiting for close lock at 1731575718898Disabling compacts and flushes for region at 1731575718898Disabling writes for close at 1731575718898Writing region close event to WAL at 1731575718898Closed at 1731575718898 2024-11-14T09:15:18,899 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T09:15:18,899 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731575718899"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731575718899"}]},"ts":"1731575718899"} 2024-11-14T09:15:18,900 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T09:15:18,901 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T09:15:18,901 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731575718901"}]},"ts":"1731575718901"} 2024-11-14T09:15:18,902 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T09:15:18,903 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3e7b42f1efdfdf4879c3f173e59339f9, ASSIGN}] 2024-11-14T09:15:18,903 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3e7b42f1efdfdf4879c3f173e59339f9, ASSIGN 2024-11-14T09:15:18,904 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3e7b42f1efdfdf4879c3f173e59339f9, ASSIGN; state=OFFLINE, location=83f56b55f2af,39927,1731575622393; forceNewPlan=false, retain=false 2024-11-14T09:15:19,055 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=3e7b42f1efdfdf4879c3f173e59339f9, regionState=OPENING, regionLocation=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:19,056 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3e7b42f1efdfdf4879c3f173e59339f9, ASSIGN because future has completed 2024-11-14T09:15:19,057 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393}] 2024-11-14T09:15:19,211 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:19,212 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7752): Opening region: {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} 2024-11-14T09:15:19,212 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:19,212 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:15:19,212 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7794): checking encryption for 3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:19,212 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7797): checking classloading for 3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:19,213 INFO [StoreOpener-3e7b42f1efdfdf4879c3f173e59339f9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:19,214 INFO [StoreOpener-3e7b42f1efdfdf4879c3f173e59339f9-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T09:15:19,214 INFO [StoreOpener-3e7b42f1efdfdf4879c3f173e59339f9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3e7b42f1efdfdf4879c3f173e59339f9 columnFamilyName A 2024-11-14T09:15:19,215 DEBUG [StoreOpener-3e7b42f1efdfdf4879c3f173e59339f9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:15:19,215 INFO [StoreOpener-3e7b42f1efdfdf4879c3f173e59339f9-1 {}] regionserver.HStore(327): Store=3e7b42f1efdfdf4879c3f173e59339f9/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:15:19,215 INFO [StoreOpener-3e7b42f1efdfdf4879c3f173e59339f9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:19,216 INFO [StoreOpener-3e7b42f1efdfdf4879c3f173e59339f9-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T09:15:19,216 INFO [StoreOpener-3e7b42f1efdfdf4879c3f173e59339f9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3e7b42f1efdfdf4879c3f173e59339f9 columnFamilyName B 2024-11-14T09:15:19,216 DEBUG [StoreOpener-3e7b42f1efdfdf4879c3f173e59339f9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:15:19,216 INFO [StoreOpener-3e7b42f1efdfdf4879c3f173e59339f9-1 {}] regionserver.HStore(327): Store=3e7b42f1efdfdf4879c3f173e59339f9/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:15:19,216 INFO [StoreOpener-3e7b42f1efdfdf4879c3f173e59339f9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:19,217 INFO [StoreOpener-3e7b42f1efdfdf4879c3f173e59339f9-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T09:15:19,217 INFO [StoreOpener-3e7b42f1efdfdf4879c3f173e59339f9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3e7b42f1efdfdf4879c3f173e59339f9 columnFamilyName C 2024-11-14T09:15:19,217 DEBUG [StoreOpener-3e7b42f1efdfdf4879c3f173e59339f9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:15:19,218 INFO [StoreOpener-3e7b42f1efdfdf4879c3f173e59339f9-1 {}] regionserver.HStore(327): Store=3e7b42f1efdfdf4879c3f173e59339f9/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:15:19,218 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:19,218 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1038): replaying wal for 3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:19,218 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:19,219 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:19,219 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1048): stopping wal replay for 3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:19,219 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1060): Cleaning up temporary data for 3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:19,220 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T09:15:19,220 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1093): writing seq id for 3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:19,222 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T09:15:19,222 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1114): Opened 3e7b42f1efdfdf4879c3f173e59339f9; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59498671, jitterRate=-0.11340071260929108}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T09:15:19,222 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:19,223 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1006): Region open journal for 3e7b42f1efdfdf4879c3f173e59339f9: Running coprocessor pre-open hook at 1731575719212Writing region info on filesystem at 1731575719212Initializing all the Stores at 1731575719213 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731575719213Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731575719213Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731575719213Cleaning up temporary data from old regions at 1731575719219 (+6 ms)Running coprocessor post-open hooks at 1731575719222 (+3 ms)Region opened successfully at 1731575719223 (+1 ms) 2024-11-14T09:15:19,223 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., pid=89, masterSystemTime=1731575719209 2024-11-14T09:15:19,225 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:19,225 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:19,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-14T09:15:19,225 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=3e7b42f1efdfdf4879c3f173e59339f9, regionState=OPEN, openSeqNum=2, regionLocation=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:19,227 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 because future has completed 2024-11-14T09:15:19,229 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-14T09:15:19,229 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; OpenRegionProcedure 3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 in 170 msec 2024-11-14T09:15:19,230 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-14T09:15:19,230 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3e7b42f1efdfdf4879c3f173e59339f9, ASSIGN in 326 msec 2024-11-14T09:15:19,231 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T09:15:19,231 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731575719231"}]},"ts":"1731575719231"} 2024-11-14T09:15:19,232 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T09:15:19,232 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T09:15:19,234 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1520 sec 2024-11-14T09:15:20,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-14T09:15:20,235 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:15:20,236 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1af6f666, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:20,236 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:15:20,237 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:15:20,238 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:15:20,238 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:15:20,238 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:15:20,238 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33fcf3a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:20,238 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:15:20,239 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:15:20,239 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:20,240 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33096, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:15:20,241 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55335f70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:20,241 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:15:20,242 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:15:20,242 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:15:20,243 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52202, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:15:20,244 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T09:15:20,245 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 83f56b55f2af,45325,1731575621660 2024-11-14T09:15:20,245 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@63882a10 2024-11-14T09:15:20,245 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T09:15:20,246 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33110, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T09:15:20,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T09:15:20,248 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T09:15:20,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-14T09:15:20,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742174_1350 (size=1097) 2024-11-14T09:15:20,664 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-14T09:15:20,664 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-14T09:15:20,668 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T09:15:20,670 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3e7b42f1efdfdf4879c3f173e59339f9, REOPEN/MOVE}] 2024-11-14T09:15:20,671 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3e7b42f1efdfdf4879c3f173e59339f9, REOPEN/MOVE 2024-11-14T09:15:20,672 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=3e7b42f1efdfdf4879c3f173e59339f9, regionState=CLOSING, regionLocation=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:20,676 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3e7b42f1efdfdf4879c3f173e59339f9, REOPEN/MOVE because future has completed 2024-11-14T09:15:20,676 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T09:15:20,676 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393}] 2024-11-14T09:15:20,829 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(122): Close 3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:20,829 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T09:15:20,829 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1722): Closing 3e7b42f1efdfdf4879c3f173e59339f9, disabling compactions & flushes 2024-11-14T09:15:20,829 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:20,829 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:20,829 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. after waiting 0 ms 2024-11-14T09:15:20,829 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:20,833 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T09:15:20,834 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:20,834 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1676): Region close journal for 3e7b42f1efdfdf4879c3f173e59339f9: Waiting for close lock at 1731575720829Running coprocessor pre-close hooks at 1731575720829Disabling compacts and flushes for region at 1731575720829Disabling writes for close at 1731575720829Writing region close event to WAL at 1731575720830 (+1 ms)Running coprocessor post-close hooks at 1731575720834 (+4 ms)Closed at 1731575720834 2024-11-14T09:15:20,834 WARN [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegionServer(3299): Not adding moved region record: 3e7b42f1efdfdf4879c3f173e59339f9 to self. 2024-11-14T09:15:20,837 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=3e7b42f1efdfdf4879c3f173e59339f9, regionState=CLOSED 2024-11-14T09:15:20,839 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(157): Closed 3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:20,839 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 because future has completed 2024-11-14T09:15:20,842 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-14T09:15:20,842 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; CloseRegionProcedure 3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 in 164 msec 2024-11-14T09:15:20,842 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3e7b42f1efdfdf4879c3f173e59339f9, REOPEN/MOVE; state=CLOSED, location=83f56b55f2af,39927,1731575622393; forceNewPlan=false, retain=true 2024-11-14T09:15:20,993 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=3e7b42f1efdfdf4879c3f173e59339f9, regionState=OPENING, regionLocation=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:20,995 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3e7b42f1efdfdf4879c3f173e59339f9, REOPEN/MOVE because future has completed 2024-11-14T09:15:20,995 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393}] 2024-11-14T09:15:21,152 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:21,152 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7752): Opening region: {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} 2024-11-14T09:15:21,152 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:21,152 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:15:21,152 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7794): checking encryption for 3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:21,152 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7797): checking classloading for 3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:21,154 INFO [StoreOpener-3e7b42f1efdfdf4879c3f173e59339f9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:21,155 INFO [StoreOpener-3e7b42f1efdfdf4879c3f173e59339f9-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T09:15:21,155 INFO [StoreOpener-3e7b42f1efdfdf4879c3f173e59339f9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3e7b42f1efdfdf4879c3f173e59339f9 columnFamilyName A 2024-11-14T09:15:21,156 DEBUG [StoreOpener-3e7b42f1efdfdf4879c3f173e59339f9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:15:21,156 INFO [StoreOpener-3e7b42f1efdfdf4879c3f173e59339f9-1 {}] regionserver.HStore(327): Store=3e7b42f1efdfdf4879c3f173e59339f9/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:15:21,157 INFO [StoreOpener-3e7b42f1efdfdf4879c3f173e59339f9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:21,157 INFO [StoreOpener-3e7b42f1efdfdf4879c3f173e59339f9-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T09:15:21,158 INFO [StoreOpener-3e7b42f1efdfdf4879c3f173e59339f9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3e7b42f1efdfdf4879c3f173e59339f9 columnFamilyName B 2024-11-14T09:15:21,158 DEBUG [StoreOpener-3e7b42f1efdfdf4879c3f173e59339f9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:15:21,158 INFO [StoreOpener-3e7b42f1efdfdf4879c3f173e59339f9-1 {}] regionserver.HStore(327): Store=3e7b42f1efdfdf4879c3f173e59339f9/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:15:21,158 INFO [StoreOpener-3e7b42f1efdfdf4879c3f173e59339f9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:21,159 INFO [StoreOpener-3e7b42f1efdfdf4879c3f173e59339f9-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T09:15:21,159 INFO [StoreOpener-3e7b42f1efdfdf4879c3f173e59339f9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3e7b42f1efdfdf4879c3f173e59339f9 columnFamilyName C 2024-11-14T09:15:21,159 DEBUG [StoreOpener-3e7b42f1efdfdf4879c3f173e59339f9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:15:21,159 INFO [StoreOpener-3e7b42f1efdfdf4879c3f173e59339f9-1 {}] regionserver.HStore(327): Store=3e7b42f1efdfdf4879c3f173e59339f9/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:15:21,160 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:21,160 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1038): replaying wal for 3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:21,160 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:21,161 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:21,162 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1048): stopping wal replay for 3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:21,162 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1060): Cleaning up temporary data for 3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:21,163 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T09:15:21,164 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1093): writing seq id for 3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:21,165 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1114): Opened 3e7b42f1efdfdf4879c3f173e59339f9; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73639941, jitterRate=0.09732063114643097}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T09:15:21,165 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:21,165 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1006): Region open journal for 3e7b42f1efdfdf4879c3f173e59339f9: Running coprocessor pre-open hook at 1731575721153Writing region info on filesystem at 1731575721153Initializing all the Stores at 1731575721153Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731575721153Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731575721154 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731575721154Cleaning up temporary data from old regions at 1731575721162 (+8 ms)Running coprocessor post-open hooks at 1731575721165 (+3 ms)Region opened successfully at 1731575721165 2024-11-14T09:15:21,166 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., pid=94, masterSystemTime=1731575721148 2024-11-14T09:15:21,168 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:21,168 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:21,168 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=3e7b42f1efdfdf4879c3f173e59339f9, regionState=OPEN, openSeqNum=5, regionLocation=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:21,170 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 because future has completed 2024-11-14T09:15:21,173 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=92 2024-11-14T09:15:21,173 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=92, state=SUCCESS, hasLock=false; OpenRegionProcedure 3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 in 176 msec 2024-11-14T09:15:21,175 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-14T09:15:21,175 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3e7b42f1efdfdf4879c3f173e59339f9, REOPEN/MOVE in 503 msec 2024-11-14T09:15:21,177 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-14T09:15:21,177 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 508 msec 2024-11-14T09:15:21,180 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 929 msec 2024-11-14T09:15:21,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-14T09:15:21,181 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:15:21,182 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9ff38b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:21,182 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:15:21,182 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:15:21,183 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:15:21,183 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:15:21,183 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:15:21,183 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cf804e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:21,183 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:15:21,183 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:15:21,183 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:21,184 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37746, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:15:21,185 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ea71e70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:21,186 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62ec6412, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:21,186 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:15:21,186 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:15:21,187 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:15:21,187 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:15:21,187 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:15:21,187 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f50f3ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:21,187 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:15:21,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:15:21,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:21,189 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37754, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:15:21,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a82228d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:21,195 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f2378c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:21,195 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:15:21,195 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:15:21,196 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:15:21,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:15:21,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:15:21,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52a0b96b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:21,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:15:21,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:15:21,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:21,197 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37768, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:15:21,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58507ef0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:21,204 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43cc49f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:21,204 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:15:21,205 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:15:21,205 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:15:21,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:15:21,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:15:21,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@547d72b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:21,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:15:21,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:15:21,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:21,207 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37790, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:15:21,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ec1122, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:21,208 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e0f0eef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:21,208 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:15:21,208 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:15:21,209 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:15:21,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:15:21,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:15:21,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@744b0679, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:21,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:15:21,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:15:21,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:21,210 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37816, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:15:21,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26166e00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:21,211 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65616af9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:21,211 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:15:21,212 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:15:21,212 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:15:21,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:15:21,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:15:21,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e314414, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:21,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:15:21,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:15:21,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:21,214 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37826, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:15:21,215 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b973fc5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:21,216 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74481d40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:21,216 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:15:21,217 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:15:21,217 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:15:21,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:15:21,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:15:21,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e101c4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:21,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:15:21,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:15:21,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:21,218 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37856, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:15:21,219 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f9dab61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:21,222 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@514b3d51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:21,222 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:15:21,222 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:15:21,223 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:15:21,223 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:15:21,223 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:15:21,223 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a8e907a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:21,223 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:15:21,224 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:15:21,224 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:21,225 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37880, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:15:21,225 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@652b9ab1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:21,235 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b14e3fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:21,235 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:15:21,235 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:15:21,236 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:15:21,236 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:15:21,236 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:15:21,237 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26484d4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:21,237 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:15:21,237 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:15:21,237 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:21,238 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37910, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:15:21,238 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77f0a639, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:21,239 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75e264c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:21,239 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:15:21,239 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:15:21,240 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:15:21,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:15:21,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:15:21,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64f84b61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:21,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:15:21,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:15:21,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:21,241 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37920, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:15:21,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@618900c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:21,242 DEBUG [Thread-1592 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:15:21,242 DEBUG [Thread-1596 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:15:21,243 DEBUG [Thread-1611 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:15:21,243 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:15:21,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:15:21,244 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:15:21,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:15:21,244 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:15:21,244 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:15:21,245 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56492, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:15:21,245 DEBUG [Thread-1609 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:15:21,246 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56502, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:15:21,246 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:15:21,246 DEBUG [Thread-1605 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:15:21,246 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:15:21,247 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56528, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:15:21,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5] 2024-11-14T09:15:21,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5] 2024-11-14T09:15:21,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5] 2024-11-14T09:15:21,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:15:21,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:15:21,252 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56544, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:15:21,253 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56516, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:15:21,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5] 2024-11-14T09:15:21,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5] 2024-11-14T09:15:21,259 DEBUG [Thread-1600 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:15:21,260 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:15:21,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:15:21,260 DEBUG [Thread-1600 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:15:21,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-14T09:15:21,261 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56556, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:15:21,261 DEBUG [Thread-1603 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:15:21,262 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:15:21,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-14T09:15:21,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:15:21,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:15:21,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5] 2024-11-14T09:15:21,264 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56572, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:15:21,264 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:15:21,264 DEBUG [Thread-1607 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:15:21,265 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:15:21,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5] 2024-11-14T09:15:21,267 DEBUG [Thread-1594 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:15:21,268 DEBUG [Thread-1607 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:15:21,268 DEBUG [Thread-1607 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:15:21,269 DEBUG [Thread-1598 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:15:21,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:15:21,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:15:21,270 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56588, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:15:21,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:15:21,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:15:21,273 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56604, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:15:21,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5] 2024-11-14T09:15:21,274 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56618, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:15:21,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5] 2024-11-14T09:15:21,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5] 2024-11-14T09:15:21,281 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3e7b42f1efdfdf4879c3f173e59339f9 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-14T09:15:21,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e7b42f1efdfdf4879c3f173e59339f9, store=A 2024-11-14T09:15:21,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:15:21,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e7b42f1efdfdf4879c3f173e59339f9, store=B 2024-11-14T09:15:21,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:15:21,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e7b42f1efdfdf4879c3f173e59339f9, store=C 2024-11-14T09:15:21,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:15:21,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on 3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:21,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411145527d028902a4516a0a53503ce38f0fb_3e7b42f1efdfdf4879c3f173e59339f9 is 50, key is test_row_0/A:col10/1731575721256/Put/seqid=0 2024-11-14T09:15:21,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742175_1351 (size=14594) 2024-11-14T09:15:21,332 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:15:21,336 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411145527d028902a4516a0a53503ce38f0fb_3e7b42f1efdfdf4879c3f173e59339f9 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145527d028902a4516a0a53503ce38f0fb_3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:21,337 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/2e05102ca2e1427dabaffc12762cce69, store: [table=TestAcidGuarantees family=A region=3e7b42f1efdfdf4879c3f173e59339f9] 2024-11-14T09:15:21,337 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/2e05102ca2e1427dabaffc12762cce69 is 175, key is test_row_0/A:col10/1731575721256/Put/seqid=0 2024-11-14T09:15:21,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:21,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56502 deadline: 1731575781326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:21,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:21,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:21,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:21,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:21,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56588 deadline: 1731575781330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:21,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:21,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:21,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:21,349 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731575721315, max timestamp=1731575721331Num uniques 859; 2024-11-14T09:15:21,349 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:15:21,349 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731575721315, max timestamp=1731575721331Num uniques 859; 2024-11-14T09:15:21,349 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:15:21,349 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731575721315, max timestamp=1731575721331Num uniques 859; 2024-11-14T09:15:21,349 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:15:21,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-14T09:15:21,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742176_1352 (size=39549) 2024-11-14T09:15:21,374 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/2e05102ca2e1427dabaffc12762cce69 2024-11-14T09:15:21,409 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/B/ff9f097805ea4ab6bce8343d3c31b98c is 50, key is test_row_0/B:col10/1731575721256/Put/seqid=0 2024-11-14T09:15:21,416 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-14T09:15:21,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:21,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. as already flushing 2024-11-14T09:15:21,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:21,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:21,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56556 deadline: 1731575781403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:21,417 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:21,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:21,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:21,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:21,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:21,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:21,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:21,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56516 deadline: 1731575781417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:21,427 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:21,427 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:21,427 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:21,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:21,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56618 deadline: 1731575781419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:21,437 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:21,437 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:21,437 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:21,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742177_1353 (size=12001) 2024-11-14T09:15:21,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:21,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56502 deadline: 1731575781455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:21,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:21,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:21,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:21,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:21,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56588 deadline: 1731575781455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:21,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:21,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:21,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:21,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:21,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56556 deadline: 1731575781525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:21,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:21,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:21,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:21,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:21,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56516 deadline: 1731575781535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:21,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:21,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:21,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:21,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:21,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56618 deadline: 1731575781546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:21,559 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:21,559 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:21,559 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:21,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-14T09:15:21,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:21,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. as already flushing 2024-11-14T09:15:21,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:21,571 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:21,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:21,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:21,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-14T09:15:21,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:21,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56502 deadline: 1731575781665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:21,670 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:21,670 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:21,670 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:21,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:21,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56588 deadline: 1731575781665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:21,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:21,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:21,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:21,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-14T09:15:21,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:21,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. as already flushing 2024-11-14T09:15:21,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:21,725 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:21,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:21,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:21,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:21,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56556 deadline: 1731575781737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:21,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:21,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:21,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:21,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:21,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56618 deadline: 1731575781765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:21,776 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:21,776 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:21,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:21,776 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:21,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56516 deadline: 1731575781765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:21,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:21,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:21,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:21,852 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/B/ff9f097805ea4ab6bce8343d3c31b98c 2024-11-14T09:15:21,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-14T09:15:21,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:21,879 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/C/45e8892f87d4461a82b1bcc88b7c5cc2 is 50, key is test_row_0/C:col10/1731575721256/Put/seqid=0 2024-11-14T09:15:21,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. as already flushing 2024-11-14T09:15:21,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:21,880 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:21,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:21,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:21,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-14T09:15:21,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742178_1354 (size=12001) 2024-11-14T09:15:21,897 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/C/45e8892f87d4461a82b1bcc88b7c5cc2 2024-11-14T09:15:21,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/2e05102ca2e1427dabaffc12762cce69 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/2e05102ca2e1427dabaffc12762cce69 2024-11-14T09:15:21,907 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/2e05102ca2e1427dabaffc12762cce69, entries=200, sequenceid=16, filesize=38.6 K 2024-11-14T09:15:21,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/B/ff9f097805ea4ab6bce8343d3c31b98c as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/ff9f097805ea4ab6bce8343d3c31b98c 2024-11-14T09:15:21,913 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/ff9f097805ea4ab6bce8343d3c31b98c, entries=150, sequenceid=16, filesize=11.7 K 2024-11-14T09:15:21,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/C/45e8892f87d4461a82b1bcc88b7c5cc2 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/45e8892f87d4461a82b1bcc88b7c5cc2 2024-11-14T09:15:21,920 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/45e8892f87d4461a82b1bcc88b7c5cc2, entries=150, sequenceid=16, filesize=11.7 K 2024-11-14T09:15:21,921 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 3e7b42f1efdfdf4879c3f173e59339f9 in 639ms, sequenceid=16, compaction requested=false 2024-11-14T09:15:21,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3e7b42f1efdfdf4879c3f173e59339f9: 2024-11-14T09:15:21,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on 3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:21,985 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3e7b42f1efdfdf4879c3f173e59339f9 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-14T09:15:21,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e7b42f1efdfdf4879c3f173e59339f9, store=A 2024-11-14T09:15:21,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:15:21,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e7b42f1efdfdf4879c3f173e59339f9, store=B 2024-11-14T09:15:21,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:15:21,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e7b42f1efdfdf4879c3f173e59339f9, store=C 2024-11-14T09:15:21,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:15:22,001 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411140121e7a0074c4ea2b59a956a41d6702a_3e7b42f1efdfdf4879c3f173e59339f9 is 50, key is test_row_0/A:col10/1731575721415/Put/seqid=0 2024-11-14T09:15:22,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742179_1355 (size=14594) 2024-11-14T09:15:22,033 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:15:22,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-14T09:15:22,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:22,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. as already flushing 2024-11-14T09:15:22,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:22,034 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:22,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:22,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:22,043 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411140121e7a0074c4ea2b59a956a41d6702a_3e7b42f1efdfdf4879c3f173e59339f9 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411140121e7a0074c4ea2b59a956a41d6702a_3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:22,044 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/380ef9e69234416cbea39c92d08284fd, store: [table=TestAcidGuarantees family=A region=3e7b42f1efdfdf4879c3f173e59339f9] 2024-11-14T09:15:22,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/380ef9e69234416cbea39c92d08284fd is 175, key is test_row_0/A:col10/1731575721415/Put/seqid=0 2024-11-14T09:15:22,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742180_1356 (size=39549) 2024-11-14T09:15:22,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:22,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56502 deadline: 1731575782066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:22,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:22,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:22,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:22,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:22,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56556 deadline: 1731575782076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:22,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:22,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:22,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:22,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:22,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56588 deadline: 1731575782082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:22,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:22,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:22,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:22,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:22,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56516 deadline: 1731575782085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:22,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:22,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:22,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:22,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:22,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56618 deadline: 1731575782085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:22,105 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:22,105 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:22,105 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:22,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-14T09:15:22,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:22,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. as already flushing 2024-11-14T09:15:22,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:22,191 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:22,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:22,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:22,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:22,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56502 deadline: 1731575782195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:22,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:22,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:22,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:22,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56588 deadline: 1731575782195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:22,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:22,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:22,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:22,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:22,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56556 deadline: 1731575782195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:22,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:22,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:22,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:22,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:22,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-14T09:15:22,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:22,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. as already flushing 2024-11-14T09:15:22,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:22,345 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:22,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:22,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:22,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-14T09:15:22,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:22,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56502 deadline: 1731575782415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:22,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:22,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56588 deadline: 1731575782415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:22,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:22,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:22,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:22,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:22,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:22,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:22,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56556 deadline: 1731575782415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:22,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:22,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:22,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:22,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:22,459 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/380ef9e69234416cbea39c92d08284fd 2024-11-14T09:15:22,467 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/B/daddea011d054a3394e9581dded8dd1b is 50, key is test_row_0/B:col10/1731575721415/Put/seqid=0 2024-11-14T09:15:22,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742181_1357 (size=12001) 2024-11-14T09:15:22,498 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-14T09:15:22,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:22,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. as already flushing 2024-11-14T09:15:22,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:22,498 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:22,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:22,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:22,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:22,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56516 deadline: 1731575782615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:22,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:22,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56618 deadline: 1731575782615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:22,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:22,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:22,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:22,619 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:22,619 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:22,619 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:22,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-14T09:15:22,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:22,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. as already flushing 2024-11-14T09:15:22,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:22,651 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:22,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:22,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:22,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:22,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56502 deadline: 1731575782735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:22,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:22,745 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:22,745 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:22,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56588 deadline: 1731575782735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:22,745 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:22,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:22,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56556 deadline: 1731575782735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:22,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:22,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:22,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:22,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:22,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:22,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:22,805 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-14T09:15:22,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:22,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. as already flushing 2024-11-14T09:15:22,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:22,805 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:22,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:22,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:22,816 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T09:15:22,870 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/B/daddea011d054a3394e9581dded8dd1b 2024-11-14T09:15:22,879 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/C/660d56825a6e42a6b50b79fd05542481 is 50, key is test_row_0/C:col10/1731575721415/Put/seqid=0 2024-11-14T09:15:22,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742182_1358 (size=12001) 2024-11-14T09:15:22,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-14T09:15:22,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:22,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. as already flushing 2024-11-14T09:15:22,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:22,958 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:22,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:22,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:23,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-14T09:15:23,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:23,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. as already flushing 2024-11-14T09:15:23,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:23,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:23,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:23,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:23,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-14T09:15:23,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:23,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. as already flushing 2024-11-14T09:15:23,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:23,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:23,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:23,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:23,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56502 deadline: 1731575783255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:23,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:23,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:23,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56588 deadline: 1731575783255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:23,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:23,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:23,266 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:23,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:23,266 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:23,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:23,266 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:23,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56556 deadline: 1731575783255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:23,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:23,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:23,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:23,283 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/C/660d56825a6e42a6b50b79fd05542481 2024-11-14T09:15:23,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/380ef9e69234416cbea39c92d08284fd as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/380ef9e69234416cbea39c92d08284fd 2024-11-14T09:15:23,291 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/380ef9e69234416cbea39c92d08284fd, entries=200, sequenceid=52, filesize=38.6 K 2024-11-14T09:15:23,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/B/daddea011d054a3394e9581dded8dd1b as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/daddea011d054a3394e9581dded8dd1b 2024-11-14T09:15:23,295 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/daddea011d054a3394e9581dded8dd1b, entries=150, sequenceid=52, filesize=11.7 K 2024-11-14T09:15:23,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/C/660d56825a6e42a6b50b79fd05542481 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/660d56825a6e42a6b50b79fd05542481 2024-11-14T09:15:23,299 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/660d56825a6e42a6b50b79fd05542481, entries=150, sequenceid=52, filesize=11.7 K 2024-11-14T09:15:23,300 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 3e7b42f1efdfdf4879c3f173e59339f9 in 1314ms, sequenceid=52, compaction requested=false 2024-11-14T09:15:23,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3e7b42f1efdfdf4879c3f173e59339f9: 2024-11-14T09:15:23,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-14T09:15:23,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-14T09:15:23,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:23,418 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 3e7b42f1efdfdf4879c3f173e59339f9 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-14T09:15:23,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e7b42f1efdfdf4879c3f173e59339f9, store=A 2024-11-14T09:15:23,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:15:23,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e7b42f1efdfdf4879c3f173e59339f9, store=B 2024-11-14T09:15:23,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:15:23,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e7b42f1efdfdf4879c3f173e59339f9, store=C 2024-11-14T09:15:23,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:15:23,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111497008c39b4cb44ad8640cbaa6a75e4a8_3e7b42f1efdfdf4879c3f173e59339f9 is 50, key is test_row_0/A:col10/1731575722024/Put/seqid=0 2024-11-14T09:15:23,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742183_1359 (size=12154) 2024-11-14T09:15:23,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on 3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:23,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. as already flushing 2024-11-14T09:15:23,706 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731575723688, max timestamp=1731575723702Num uniques 859; 2024-11-14T09:15:23,706 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T09:15:23,706 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731575723688, max timestamp=1731575723702Num uniques 859; 2024-11-14T09:15:23,706 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731575723688, max timestamp=1731575723702Num uniques 859; 2024-11-14T09:15:23,706 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T09:15:23,706 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T09:15:23,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:23,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56516 deadline: 1731575783770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:23,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:23,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56618 deadline: 1731575783782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:23,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:23,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:23,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:23,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:23,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:23,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:23,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:15:23,836 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111497008c39b4cb44ad8640cbaa6a75e4a8_3e7b42f1efdfdf4879c3f173e59339f9 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111497008c39b4cb44ad8640cbaa6a75e4a8_3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:23,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/d222198bbbe943a78fc9c48d98443291, store: [table=TestAcidGuarantees family=A region=3e7b42f1efdfdf4879c3f173e59339f9] 2024-11-14T09:15:23,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/d222198bbbe943a78fc9c48d98443291 is 175, key is test_row_0/A:col10/1731575722024/Put/seqid=0 2024-11-14T09:15:23,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742184_1360 (size=30955) 2024-11-14T09:15:23,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:23,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56516 deadline: 1731575783895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:23,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:23,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:23,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:23,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:23,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56618 deadline: 1731575783895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:23,907 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:23,907 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:23,907 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:24,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:24,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56516 deadline: 1731575784115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:24,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:24,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:24,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:24,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56618 deadline: 1731575784115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:24,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:24,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:24,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:24,121 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:24,241 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=63, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/d222198bbbe943a78fc9c48d98443291 2024-11-14T09:15:24,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/B/df5979a7878c440c891db862a393d0f2 is 50, key is test_row_0/B:col10/1731575722024/Put/seqid=0 2024-11-14T09:15:24,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742185_1361 (size=12001) 2024-11-14T09:15:24,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:24,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56588 deadline: 1731575784275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:24,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:24,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56556 deadline: 1731575784275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:24,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:24,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56502 deadline: 1731575784275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:24,284 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_1' in 3e7b42f1efdfdf4879c3f173e59339f9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:15:24,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:24,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:24,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:24,284 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_0' in 3e7b42f1efdfdf4879c3f173e59339f9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:15:24,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:24,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:24,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:24,285 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_1' in 3e7b42f1efdfdf4879c3f173e59339f9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:15:24,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:24,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:24,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:24,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:24,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56516 deadline: 1731575784425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:24,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:24,436 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:24,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56618 deadline: 1731575784425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:24,436 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:24,436 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:24,436 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:24,436 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:24,436 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:24,653 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/B/df5979a7878c440c891db862a393d0f2 2024-11-14T09:15:24,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/C/8c82de3224454519b6c104ce8a63ae70 is 50, key is test_row_0/C:col10/1731575722024/Put/seqid=0 2024-11-14T09:15:24,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742186_1362 (size=12001) 2024-11-14T09:15:24,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:24,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56516 deadline: 1731575784945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:24,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:24,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:24,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:24,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56618 deadline: 1731575784945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:24,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:24,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:24,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:24,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:25,065 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/C/8c82de3224454519b6c104ce8a63ae70 2024-11-14T09:15:25,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/d222198bbbe943a78fc9c48d98443291 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/d222198bbbe943a78fc9c48d98443291 2024-11-14T09:15:25,074 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/d222198bbbe943a78fc9c48d98443291, entries=150, sequenceid=63, filesize=30.2 K 2024-11-14T09:15:25,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/B/df5979a7878c440c891db862a393d0f2 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/df5979a7878c440c891db862a393d0f2 2024-11-14T09:15:25,078 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/df5979a7878c440c891db862a393d0f2, entries=150, sequenceid=63, filesize=11.7 K 2024-11-14T09:15:25,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/C/8c82de3224454519b6c104ce8a63ae70 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/8c82de3224454519b6c104ce8a63ae70 2024-11-14T09:15:25,086 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/8c82de3224454519b6c104ce8a63ae70, entries=150, sequenceid=63, filesize=11.7 K 2024-11-14T09:15:25,087 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 3e7b42f1efdfdf4879c3f173e59339f9 in 1669ms, sequenceid=63, compaction requested=true 2024-11-14T09:15:25,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 3e7b42f1efdfdf4879c3f173e59339f9: 2024-11-14T09:15:25,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:25,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-14T09:15:25,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-14T09:15:25,090 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-14T09:15:25,090 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.8250 sec 2024-11-14T09:15:25,092 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 3.8310 sec 2024-11-14T09:15:25,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-14T09:15:25,415 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:15:25,415 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:15:25,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-14T09:15:25,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-14T09:15:25,417 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:15:25,418 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:15:25,418 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:15:25,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-14T09:15:25,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-14T09:15:25,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:25,570 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 3e7b42f1efdfdf4879c3f173e59339f9 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-14T09:15:25,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e7b42f1efdfdf4879c3f173e59339f9, store=A 2024-11-14T09:15:25,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:15:25,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e7b42f1efdfdf4879c3f173e59339f9, store=B 2024-11-14T09:15:25,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:15:25,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e7b42f1efdfdf4879c3f173e59339f9, store=C 2024-11-14T09:15:25,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:15:25,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114fbded4556f0949f0af602f76cc3935b5_3e7b42f1efdfdf4879c3f173e59339f9 is 50, key is test_row_0/A:col10/1731575723767/Put/seqid=0 2024-11-14T09:15:25,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742187_1363 (size=12154) 2024-11-14T09:15:25,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-14T09:15:25,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on 3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:25,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. as already flushing 2024-11-14T09:15:25,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:15:25,987 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114fbded4556f0949f0af602f76cc3935b5_3e7b42f1efdfdf4879c3f173e59339f9 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114fbded4556f0949f0af602f76cc3935b5_3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:25,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/6ae67e5ade1246fbac5081fa0249fd1d, store: [table=TestAcidGuarantees family=A region=3e7b42f1efdfdf4879c3f173e59339f9] 2024-11-14T09:15:25,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/6ae67e5ade1246fbac5081fa0249fd1d is 175, key is test_row_0/A:col10/1731575723767/Put/seqid=0 2024-11-14T09:15:25,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742188_1364 (size=30955) 2024-11-14T09:15:26,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:26,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56516 deadline: 1731575786000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:26,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:26,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:26,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:26,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:26,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56618 deadline: 1731575786004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:26,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:26,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:26,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:26,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-14T09:15:26,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:26,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56516 deadline: 1731575786115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:26,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:26,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:26,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:26,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:26,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56618 deadline: 1731575786124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:26,139 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:26,139 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:26,139 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:26,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:26,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56588 deadline: 1731575786295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:26,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:26,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56556 deadline: 1731575786295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:26,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:26,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56502 deadline: 1731575786295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:26,305 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_1' in 3e7b42f1efdfdf4879c3f173e59339f9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:15:26,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:26,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:26,306 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_0' in 3e7b42f1efdfdf4879c3f173e59339f9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4229 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:15:26,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:26,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:26,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:26,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:26,306 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_1' in 3e7b42f1efdfdf4879c3f173e59339f9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4240 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:15:26,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:26,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:26,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:26,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:26,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56516 deadline: 1731575786335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:26,345 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:26,345 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:26,345 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:26,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:26,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56618 deadline: 1731575786345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:26,353 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:26,353 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:26,353 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:26,394 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=98, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/6ae67e5ade1246fbac5081fa0249fd1d 2024-11-14T09:15:26,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/B/e4779a7fb55f4615bb7a903b88502c26 is 50, key is test_row_0/B:col10/1731575723767/Put/seqid=0 2024-11-14T09:15:26,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742189_1365 (size=12001) 2024-11-14T09:15:26,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-14T09:15:26,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:26,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56516 deadline: 1731575786654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:26,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:26,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:26,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:26,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56618 deadline: 1731575786654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:26,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:26,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:26,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:26,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:26,806 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/B/e4779a7fb55f4615bb7a903b88502c26 2024-11-14T09:15:26,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/C/58f7cbb3f54b424cb271771435f75a5e is 50, key is test_row_0/C:col10/1731575723767/Put/seqid=0 2024-11-14T09:15:26,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742190_1366 (size=12001) 2024-11-14T09:15:27,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:27,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56516 deadline: 1731575787165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:27,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:27,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:27,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:27,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:27,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56618 deadline: 1731575787165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:27,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:27,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:27,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:27,218 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/C/58f7cbb3f54b424cb271771435f75a5e 2024-11-14T09:15:27,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/6ae67e5ade1246fbac5081fa0249fd1d as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/6ae67e5ade1246fbac5081fa0249fd1d 2024-11-14T09:15:27,229 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/6ae67e5ade1246fbac5081fa0249fd1d, entries=150, sequenceid=98, filesize=30.2 K 2024-11-14T09:15:27,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/B/e4779a7fb55f4615bb7a903b88502c26 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/e4779a7fb55f4615bb7a903b88502c26 2024-11-14T09:15:27,233 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/e4779a7fb55f4615bb7a903b88502c26, entries=150, sequenceid=98, filesize=11.7 K 2024-11-14T09:15:27,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/C/58f7cbb3f54b424cb271771435f75a5e as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/58f7cbb3f54b424cb271771435f75a5e 2024-11-14T09:15:27,237 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/58f7cbb3f54b424cb271771435f75a5e, entries=150, sequenceid=98, filesize=11.7 K 2024-11-14T09:15:27,238 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 3e7b42f1efdfdf4879c3f173e59339f9 in 1668ms, sequenceid=98, compaction requested=true 2024-11-14T09:15:27,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 3e7b42f1efdfdf4879c3f173e59339f9: 2024-11-14T09:15:27,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:27,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-14T09:15:27,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-14T09:15:27,241 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-14T09:15:27,241 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8210 sec 2024-11-14T09:15:27,242 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.8260 sec 2024-11-14T09:15:27,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-14T09:15:27,565 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:15:27,566 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:15:27,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-14T09:15:27,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-14T09:15:27,568 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:15:27,569 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:15:27,569 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:15:27,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-14T09:15:27,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-14T09:15:27,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:27,723 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 3e7b42f1efdfdf4879c3f173e59339f9 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-14T09:15:27,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e7b42f1efdfdf4879c3f173e59339f9, store=A 2024-11-14T09:15:27,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:15:27,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e7b42f1efdfdf4879c3f173e59339f9, store=B 2024-11-14T09:15:27,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:15:27,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e7b42f1efdfdf4879c3f173e59339f9, store=C 2024-11-14T09:15:27,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:15:27,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e8fc6d2adab2478fa4d02c54c7a8e462_3e7b42f1efdfdf4879c3f173e59339f9 is 50, key is test_row_0/A:col10/1731575725988/Put/seqid=0 2024-11-14T09:15:27,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742191_1367 (size=12154) 2024-11-14T09:15:27,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-14T09:15:28,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:15:28,143 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e8fc6d2adab2478fa4d02c54c7a8e462_3e7b42f1efdfdf4879c3f173e59339f9 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e8fc6d2adab2478fa4d02c54c7a8e462_3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:28,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/385d8245cf934bcf87169ff16dac88d6, store: [table=TestAcidGuarantees family=A region=3e7b42f1efdfdf4879c3f173e59339f9] 2024-11-14T09:15:28,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/385d8245cf934bcf87169ff16dac88d6 is 175, key is test_row_0/A:col10/1731575725988/Put/seqid=0 2024-11-14T09:15:28,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742192_1368 (size=30955) 2024-11-14T09:15:28,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on 3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:28,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. as already flushing 2024-11-14T09:15:28,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-14T09:15:28,270 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731575728203, max timestamp=1731575728266Num uniques 859; 2024-11-14T09:15:28,270 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T09:15:28,270 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731575728203, max timestamp=1731575728266Num uniques 859; 2024-11-14T09:15:28,271 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T09:15:28,272 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731575728203, max timestamp=1731575728266Num uniques 859; 2024-11-14T09:15:28,272 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T09:15:28,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:28,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56618 deadline: 1731575788361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:28,372 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:28,373 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:28,373 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:28,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:28,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56516 deadline: 1731575788367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:28,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:28,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:28,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:28,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:28,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56618 deadline: 1731575788474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:28,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:28,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:28,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:28,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:28,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56516 deadline: 1731575788485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:28,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:28,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:28,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:28,550 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=109, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/385d8245cf934bcf87169ff16dac88d6 2024-11-14T09:15:28,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/B/5e5185c4ecc846d69e929395891d25da is 50, key is test_row_0/B:col10/1731575725988/Put/seqid=0 2024-11-14T09:15:28,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742193_1369 (size=12001) 2024-11-14T09:15:28,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-14T09:15:28,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:28,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56618 deadline: 1731575788694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:28,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:28,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:28,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:28,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56516 deadline: 1731575788705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:28,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:28,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:28,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:28,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:28,961 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/B/5e5185c4ecc846d69e929395891d25da 2024-11-14T09:15:28,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/C/14907c6607f7409da44ed7257aa43bab is 50, key is test_row_0/C:col10/1731575725988/Put/seqid=0 2024-11-14T09:15:28,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742194_1370 (size=12001) 2024-11-14T09:15:29,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:29,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56618 deadline: 1731575789025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:29,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:29,038 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:29,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56516 deadline: 1731575789025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:29,038 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:29,038 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:29,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:29,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:29,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:29,382 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/C/14907c6607f7409da44ed7257aa43bab 2024-11-14T09:15:29,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/385d8245cf934bcf87169ff16dac88d6 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/385d8245cf934bcf87169ff16dac88d6 2024-11-14T09:15:29,390 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/385d8245cf934bcf87169ff16dac88d6, entries=150, sequenceid=109, filesize=30.2 K 2024-11-14T09:15:29,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/B/5e5185c4ecc846d69e929395891d25da as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/5e5185c4ecc846d69e929395891d25da 2024-11-14T09:15:29,395 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/5e5185c4ecc846d69e929395891d25da, entries=150, sequenceid=109, filesize=11.7 K 2024-11-14T09:15:29,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/C/14907c6607f7409da44ed7257aa43bab as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/14907c6607f7409da44ed7257aa43bab 2024-11-14T09:15:29,398 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/14907c6607f7409da44ed7257aa43bab, entries=150, sequenceid=109, filesize=11.7 K 2024-11-14T09:15:29,399 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 3e7b42f1efdfdf4879c3f173e59339f9 in 1677ms, sequenceid=109, compaction requested=true 2024-11-14T09:15:29,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 3e7b42f1efdfdf4879c3f173e59339f9: 2024-11-14T09:15:29,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:29,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-14T09:15:29,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-14T09:15:29,402 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-14T09:15:29,402 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8310 sec 2024-11-14T09:15:29,404 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.8370 sec 2024-11-14T09:15:29,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on 3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:29,556 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3e7b42f1efdfdf4879c3f173e59339f9 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-14T09:15:29,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e7b42f1efdfdf4879c3f173e59339f9, store=A 2024-11-14T09:15:29,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:15:29,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e7b42f1efdfdf4879c3f173e59339f9, store=B 2024-11-14T09:15:29,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:15:29,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e7b42f1efdfdf4879c3f173e59339f9, store=C 2024-11-14T09:15:29,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:15:29,563 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411145313ba8666cc4e079ec2de37041caf0b_3e7b42f1efdfdf4879c3f173e59339f9 is 50, key is test_row_0/A:col10/1731575728348/Put/seqid=0 2024-11-14T09:15:29,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742195_1371 (size=14794) 2024-11-14T09:15:29,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:29,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56516 deadline: 1731575789595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:29,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:29,617 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:29,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:29,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56618 deadline: 1731575789606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:29,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:29,618 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:29,618 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:29,618 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:29,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-14T09:15:29,716 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:15:29,716 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:15:29,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-14T09:15:29,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-14T09:15:29,718 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:15:29,718 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:15:29,719 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:15:29,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:29,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56516 deadline: 1731575789724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:29,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:29,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:29,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56618 deadline: 1731575789724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:29,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:29,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:29,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:29,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:29,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:29,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-14T09:15:29,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-14T09:15:29,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:29,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. as already flushing 2024-11-14T09:15:29,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:29,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:29,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:29,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:29,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:29,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56516 deadline: 1731575789945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:29,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:29,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:29,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56618 deadline: 1731575789945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:29,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:29,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:29,958 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:29,958 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:29,958 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:29,969 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:15:29,973 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411145313ba8666cc4e079ec2de37041caf0b_3e7b42f1efdfdf4879c3f173e59339f9 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145313ba8666cc4e079ec2de37041caf0b_3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:29,973 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/2cb74e9db25d4c77adb4e29a612c755c, store: [table=TestAcidGuarantees family=A region=3e7b42f1efdfdf4879c3f173e59339f9] 2024-11-14T09:15:29,974 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/2cb74e9db25d4c77adb4e29a612c755c is 175, key is test_row_0/A:col10/1731575728348/Put/seqid=0 2024-11-14T09:15:29,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742196_1372 (size=39749) 2024-11-14T09:15:29,978 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=145, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/2cb74e9db25d4c77adb4e29a612c755c 2024-11-14T09:15:29,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/B/66a303432a87471aa7cc19548d1e3bdc is 50, key is test_row_0/B:col10/1731575728348/Put/seqid=0 2024-11-14T09:15:29,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742197_1373 (size=12151) 2024-11-14T09:15:29,990 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/B/66a303432a87471aa7cc19548d1e3bdc 2024-11-14T09:15:29,997 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/C/c4a4e87d25f44ada8cc09f7b6c3912e5 is 50, key is test_row_0/C:col10/1731575728348/Put/seqid=0 2024-11-14T09:15:30,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742198_1374 (size=12151) 2024-11-14T09:15:30,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-14T09:15:30,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:30,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. as already flushing 2024-11-14T09:15:30,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:30,024 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:30,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:30,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:30,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-14T09:15:30,177 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-14T09:15:30,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:30,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. as already flushing 2024-11-14T09:15:30,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:30,177 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:30,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:30,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:30,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:30,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56516 deadline: 1731575790265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:30,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:30,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56618 deadline: 1731575790265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:30,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:30,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:30,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:30,279 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:30,279 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:30,279 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:30,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-14T09:15:30,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:30,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:30,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. as already flushing 2024-11-14T09:15:30,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56502 deadline: 1731575790315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:30,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:30,330 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:30,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:30,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:30,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56588 deadline: 1731575790324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:30,331 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_1' in 3e7b42f1efdfdf4879c3f173e59339f9 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8265 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:15:30,331 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:30,331 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:30,331 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:30,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:30,331 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_1' in 3e7b42f1efdfdf4879c3f173e59339f9 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8248 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:15:30,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:30,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:30,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:30,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-14T09:15:30,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:30,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56556 deadline: 1731575790345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:30,351 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_0' in 3e7b42f1efdfdf4879c3f173e59339f9 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8275 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:15:30,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:30,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:30,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:30,401 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/C/c4a4e87d25f44ada8cc09f7b6c3912e5 2024-11-14T09:15:30,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/2cb74e9db25d4c77adb4e29a612c755c as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/2cb74e9db25d4c77adb4e29a612c755c 2024-11-14T09:15:30,407 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/2cb74e9db25d4c77adb4e29a612c755c, entries=200, sequenceid=145, filesize=38.8 K 2024-11-14T09:15:30,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/B/66a303432a87471aa7cc19548d1e3bdc as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/66a303432a87471aa7cc19548d1e3bdc 2024-11-14T09:15:30,411 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/66a303432a87471aa7cc19548d1e3bdc, entries=150, sequenceid=145, filesize=11.9 K 2024-11-14T09:15:30,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/C/c4a4e87d25f44ada8cc09f7b6c3912e5 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/c4a4e87d25f44ada8cc09f7b6c3912e5 2024-11-14T09:15:30,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/c4a4e87d25f44ada8cc09f7b6c3912e5, entries=150, sequenceid=145, filesize=11.9 K 2024-11-14T09:15:30,416 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 3e7b42f1efdfdf4879c3f173e59339f9 in 860ms, sequenceid=145, compaction requested=true 2024-11-14T09:15:30,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3e7b42f1efdfdf4879c3f173e59339f9: 2024-11-14T09:15:30,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3e7b42f1efdfdf4879c3f173e59339f9:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:15:30,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:15:30,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3e7b42f1efdfdf4879c3f173e59339f9:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:15:30,416 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-14T09:15:30,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:15:30,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3e7b42f1efdfdf4879c3f173e59339f9:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:15:30,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T09:15:30,416 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-14T09:15:30,418 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 211712 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-14T09:15:30,418 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 72156 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-14T09:15:30,418 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): 3e7b42f1efdfdf4879c3f173e59339f9/A is initiating minor compaction (all files) 2024-11-14T09:15:30,418 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): 3e7b42f1efdfdf4879c3f173e59339f9/B is initiating minor compaction (all files) 2024-11-14T09:15:30,418 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3e7b42f1efdfdf4879c3f173e59339f9/A in TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:30,418 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/2e05102ca2e1427dabaffc12762cce69, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/380ef9e69234416cbea39c92d08284fd, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/d222198bbbe943a78fc9c48d98443291, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/6ae67e5ade1246fbac5081fa0249fd1d, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/385d8245cf934bcf87169ff16dac88d6, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/2cb74e9db25d4c77adb4e29a612c755c] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp, totalSize=206.8 K 2024-11-14T09:15:30,418 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=10 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:30,418 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. files: [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/2e05102ca2e1427dabaffc12762cce69, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/380ef9e69234416cbea39c92d08284fd, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/d222198bbbe943a78fc9c48d98443291, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/6ae67e5ade1246fbac5081fa0249fd1d, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/385d8245cf934bcf87169ff16dac88d6, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/2cb74e9db25d4c77adb4e29a612c755c] 2024-11-14T09:15:30,418 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3e7b42f1efdfdf4879c3f173e59339f9/B in TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:30,418 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/ff9f097805ea4ab6bce8343d3c31b98c, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/daddea011d054a3394e9581dded8dd1b, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/df5979a7878c440c891db862a393d0f2, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/e4779a7fb55f4615bb7a903b88502c26, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/5e5185c4ecc846d69e929395891d25da, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/66a303432a87471aa7cc19548d1e3bdc] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp, totalSize=70.5 K 2024-11-14T09:15:30,419 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e05102ca2e1427dabaffc12762cce69, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731575721256 2024-11-14T09:15:30,419 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting ff9f097805ea4ab6bce8343d3c31b98c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731575721256 2024-11-14T09:15:30,419 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting daddea011d054a3394e9581dded8dd1b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731575721409 2024-11-14T09:15:30,419 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 380ef9e69234416cbea39c92d08284fd, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731575721391 2024-11-14T09:15:30,419 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting df5979a7878c440c891db862a393d0f2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731575722024 2024-11-14T09:15:30,419 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting d222198bbbe943a78fc9c48d98443291, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731575722024 2024-11-14T09:15:30,420 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting e4779a7fb55f4615bb7a903b88502c26, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731575723767 2024-11-14T09:15:30,420 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ae67e5ade1246fbac5081fa0249fd1d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731575723767 2024-11-14T09:15:30,420 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e5185c4ecc846d69e929395891d25da, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1731575725984 2024-11-14T09:15:30,420 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 385d8245cf934bcf87169ff16dac88d6, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1731575725984 2024-11-14T09:15:30,420 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 66a303432a87471aa7cc19548d1e3bdc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731575728348 2024-11-14T09:15:30,420 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2cb74e9db25d4c77adb4e29a612c755c, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731575728348 2024-11-14T09:15:30,430 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3e7b42f1efdfdf4879c3f173e59339f9] 2024-11-14T09:15:30,432 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114b083e2e8ce024101a5ea387a287f60e8_3e7b42f1efdfdf4879c3f173e59339f9 store=[table=TestAcidGuarantees family=A region=3e7b42f1efdfdf4879c3f173e59339f9] 2024-11-14T09:15:30,436 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114b083e2e8ce024101a5ea387a287f60e8_3e7b42f1efdfdf4879c3f173e59339f9, store=[table=TestAcidGuarantees family=A region=3e7b42f1efdfdf4879c3f173e59339f9] 2024-11-14T09:15:30,436 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b083e2e8ce024101a5ea387a287f60e8_3e7b42f1efdfdf4879c3f173e59339f9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3e7b42f1efdfdf4879c3f173e59339f9] 2024-11-14T09:15:30,440 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3e7b42f1efdfdf4879c3f173e59339f9#B#compaction#319 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:15:30,440 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/B/0e48131d01d143eca9996e938a879208 is 50, key is test_row_0/B:col10/1731575728348/Put/seqid=0 2024-11-14T09:15:30,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742199_1375 (size=4469) 2024-11-14T09:15:30,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742200_1376 (size=12357) 2024-11-14T09:15:30,453 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3e7b42f1efdfdf4879c3f173e59339f9#A#compaction#318 average throughput is 1.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:15:30,454 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/d649935043724b438195a8817ff9868c is 175, key is test_row_0/A:col10/1731575728348/Put/seqid=0 2024-11-14T09:15:30,459 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/B/0e48131d01d143eca9996e938a879208 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/0e48131d01d143eca9996e938a879208 2024-11-14T09:15:30,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742201_1377 (size=31311) 2024-11-14T09:15:30,465 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/d649935043724b438195a8817ff9868c as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/d649935043724b438195a8817ff9868c 2024-11-14T09:15:30,467 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 3e7b42f1efdfdf4879c3f173e59339f9/B of 3e7b42f1efdfdf4879c3f173e59339f9 into 0e48131d01d143eca9996e938a879208(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:15:30,467 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3e7b42f1efdfdf4879c3f173e59339f9: 2024-11-14T09:15:30,467 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., storeName=3e7b42f1efdfdf4879c3f173e59339f9/B, priority=10, startTime=1731575730416; duration=0sec 2024-11-14T09:15:30,467 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:15:30,467 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3e7b42f1efdfdf4879c3f173e59339f9:B 2024-11-14T09:15:30,467 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-14T09:15:30,470 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 3e7b42f1efdfdf4879c3f173e59339f9/A of 3e7b42f1efdfdf4879c3f173e59339f9 into d649935043724b438195a8817ff9868c(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:15:30,470 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3e7b42f1efdfdf4879c3f173e59339f9: 2024-11-14T09:15:30,470 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., storeName=3e7b42f1efdfdf4879c3f173e59339f9/A, priority=10, startTime=1731575730416; duration=0sec 2024-11-14T09:15:30,470 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:15:30,470 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3e7b42f1efdfdf4879c3f173e59339f9:A 2024-11-14T09:15:30,470 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 72156 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-14T09:15:30,470 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): 3e7b42f1efdfdf4879c3f173e59339f9/C is initiating minor compaction (all files) 2024-11-14T09:15:30,470 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3e7b42f1efdfdf4879c3f173e59339f9/C in TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:30,470 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/45e8892f87d4461a82b1bcc88b7c5cc2, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/660d56825a6e42a6b50b79fd05542481, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/8c82de3224454519b6c104ce8a63ae70, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/58f7cbb3f54b424cb271771435f75a5e, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/14907c6607f7409da44ed7257aa43bab, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/c4a4e87d25f44ada8cc09f7b6c3912e5] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp, totalSize=70.5 K 2024-11-14T09:15:30,471 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 45e8892f87d4461a82b1bcc88b7c5cc2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731575721256 2024-11-14T09:15:30,471 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 660d56825a6e42a6b50b79fd05542481, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731575721409 2024-11-14T09:15:30,471 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c82de3224454519b6c104ce8a63ae70, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731575722024 2024-11-14T09:15:30,472 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 58f7cbb3f54b424cb271771435f75a5e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731575723767 2024-11-14T09:15:30,472 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 14907c6607f7409da44ed7257aa43bab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1731575725984 2024-11-14T09:15:30,472 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting c4a4e87d25f44ada8cc09f7b6c3912e5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731575728348 2024-11-14T09:15:30,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-14T09:15:30,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:30,484 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 3e7b42f1efdfdf4879c3f173e59339f9 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T09:15:30,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e7b42f1efdfdf4879c3f173e59339f9, store=A 2024-11-14T09:15:30,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:15:30,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e7b42f1efdfdf4879c3f173e59339f9, store=B 2024-11-14T09:15:30,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:15:30,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e7b42f1efdfdf4879c3f173e59339f9, store=C 2024-11-14T09:15:30,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:15:30,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111441cdc958f94549788203656293259a0b_3e7b42f1efdfdf4879c3f173e59339f9 is 50, key is test_row_0/A:col10/1731575729593/Put/seqid=0 2024-11-14T09:15:30,491 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3e7b42f1efdfdf4879c3f173e59339f9#C#compaction#320 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:15:30,492 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/C/d76752d352984488bea5d6725bc1619f is 50, key is test_row_0/C:col10/1731575728348/Put/seqid=0 2024-11-14T09:15:30,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742202_1378 (size=12304) 2024-11-14T09:15:30,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742203_1379 (size=12357) 2024-11-14T09:15:30,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on 3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:30,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. as already flushing 2024-11-14T09:15:30,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-14T09:15:30,865 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731575730843, max timestamp=1731575730862Num uniques 859; 2024-11-14T09:15:30,865 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T09:15:30,865 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731575730843, max timestamp=1731575730862Num uniques 859; 2024-11-14T09:15:30,865 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731575730843, max timestamp=1731575730862Num uniques 859; 2024-11-14T09:15:30,865 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T09:15:30,865 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T09:15:30,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:15:30,899 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111441cdc958f94549788203656293259a0b_3e7b42f1efdfdf4879c3f173e59339f9 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111441cdc958f94549788203656293259a0b_3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:30,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/fc638de742f64c3e9eef03fe047e63df, store: [table=TestAcidGuarantees family=A region=3e7b42f1efdfdf4879c3f173e59339f9] 2024-11-14T09:15:30,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/fc638de742f64c3e9eef03fe047e63df is 175, key is test_row_0/A:col10/1731575729593/Put/seqid=0 2024-11-14T09:15:30,905 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/C/d76752d352984488bea5d6725bc1619f as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/d76752d352984488bea5d6725bc1619f 2024-11-14T09:15:30,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742204_1380 (size=31105) 2024-11-14T09:15:30,909 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 3e7b42f1efdfdf4879c3f173e59339f9/C of 3e7b42f1efdfdf4879c3f173e59339f9 into d76752d352984488bea5d6725bc1619f(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:15:30,909 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3e7b42f1efdfdf4879c3f173e59339f9: 2024-11-14T09:15:30,909 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., storeName=3e7b42f1efdfdf4879c3f173e59339f9/C, priority=10, startTime=1731575730416; duration=0sec 2024-11-14T09:15:30,909 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:15:30,909 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3e7b42f1efdfdf4879c3f173e59339f9:C 2024-11-14T09:15:30,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:30,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56516 deadline: 1731575790934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:30,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:30,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:30,950 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:30,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:30,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56618 deadline: 1731575790949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:30,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:30,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:30,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:31,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:31,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56516 deadline: 1731575791054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:31,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:31,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:31,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:31,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:31,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56618 deadline: 1731575791065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:31,072 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:31,072 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:31,072 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:31,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:31,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56516 deadline: 1731575791275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:31,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:31,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:31,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:31,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:31,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56618 deadline: 1731575791275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:31,285 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:31,285 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:31,285 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:31,307 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=157, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/fc638de742f64c3e9eef03fe047e63df 2024-11-14T09:15:31,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/B/3314466ada124ff0a1d3c7b7b8f70870 is 50, key is test_row_0/B:col10/1731575729593/Put/seqid=0 2024-11-14T09:15:31,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742205_1381 (size=12151) 2024-11-14T09:15:31,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:31,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56516 deadline: 1731575791595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:31,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:31,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:31,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:31,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:31,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56618 deadline: 1731575791595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:31,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:31,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:31,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:31,723 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/B/3314466ada124ff0a1d3c7b7b8f70870 2024-11-14T09:15:31,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/C/6a3e9821aae74556a4ee90fa639965f2 is 50, key is test_row_0/C:col10/1731575729593/Put/seqid=0 2024-11-14T09:15:31,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742206_1382 (size=12151) 2024-11-14T09:15:31,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-14T09:15:32,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:32,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56516 deadline: 1731575792105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:32,116 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:32,116 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:32,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:32,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:32,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56618 deadline: 1731575792114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:32,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:32,121 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:32,121 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:32,137 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/C/6a3e9821aae74556a4ee90fa639965f2 2024-11-14T09:15:32,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/fc638de742f64c3e9eef03fe047e63df as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/fc638de742f64c3e9eef03fe047e63df 2024-11-14T09:15:32,149 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/fc638de742f64c3e9eef03fe047e63df, entries=150, sequenceid=157, filesize=30.4 K 2024-11-14T09:15:32,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/B/3314466ada124ff0a1d3c7b7b8f70870 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/3314466ada124ff0a1d3c7b7b8f70870 2024-11-14T09:15:32,159 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/3314466ada124ff0a1d3c7b7b8f70870, entries=150, sequenceid=157, filesize=11.9 K 2024-11-14T09:15:32,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/C/6a3e9821aae74556a4ee90fa639965f2 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/6a3e9821aae74556a4ee90fa639965f2 2024-11-14T09:15:32,164 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/6a3e9821aae74556a4ee90fa639965f2, entries=150, sequenceid=157, filesize=11.9 K 2024-11-14T09:15:32,165 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 3e7b42f1efdfdf4879c3f173e59339f9 in 1681ms, sequenceid=157, compaction requested=false 2024-11-14T09:15:32,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 3e7b42f1efdfdf4879c3f173e59339f9: 2024-11-14T09:15:32,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:32,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-14T09:15:32,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-14T09:15:32,170 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-14T09:15:32,170 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4480 sec 2024-11-14T09:15:32,174 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.4550 sec 2024-11-14T09:15:33,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on 3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:33,144 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3e7b42f1efdfdf4879c3f173e59339f9 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-14T09:15:33,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e7b42f1efdfdf4879c3f173e59339f9, store=A 2024-11-14T09:15:33,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:15:33,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e7b42f1efdfdf4879c3f173e59339f9, store=B 2024-11-14T09:15:33,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:15:33,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e7b42f1efdfdf4879c3f173e59339f9, store=C 2024-11-14T09:15:33,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:15:33,152 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111426d16803292e4c4091f7b5c2a8905881_3e7b42f1efdfdf4879c3f173e59339f9 is 50, key is test_row_0/A:col10/1731575730921/Put/seqid=0 2024-11-14T09:15:33,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742207_1383 (size=14794) 2024-11-14T09:15:33,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:33,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56516 deadline: 1731575793175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:33,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:33,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:33,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:33,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:33,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56618 deadline: 1731575793177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:33,190 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:33,190 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:33,190 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:33,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:33,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56516 deadline: 1731575793285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:33,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:33,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:33,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:33,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:33,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56618 deadline: 1731575793295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:33,306 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:33,306 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:33,306 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:33,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:33,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56516 deadline: 1731575793505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:33,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:33,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:33,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:33,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:33,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56618 deadline: 1731575793515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:33,521 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:33,521 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:33,521 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:33,562 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:15:33,565 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111426d16803292e4c4091f7b5c2a8905881_3e7b42f1efdfdf4879c3f173e59339f9 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111426d16803292e4c4091f7b5c2a8905881_3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:33,566 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/13ffeb1b07834f8797a073fa82ae7e30, store: [table=TestAcidGuarantees family=A region=3e7b42f1efdfdf4879c3f173e59339f9] 2024-11-14T09:15:33,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/13ffeb1b07834f8797a073fa82ae7e30 is 175, key is test_row_0/A:col10/1731575730921/Put/seqid=0 2024-11-14T09:15:33,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742208_1384 (size=39749) 2024-11-14T09:15:33,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:33,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56516 deadline: 1731575793825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:33,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:33,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56618 deadline: 1731575793825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:33,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:33,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:33,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:33,832 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:33,832 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:33,832 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:33,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-14T09:15:33,905 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:15:33,906 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:15:33,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-14T09:15:33,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-14T09:15:33,907 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:15:33,908 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:15:33,908 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:15:33,971 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=194, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/13ffeb1b07834f8797a073fa82ae7e30 2024-11-14T09:15:33,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/B/e8d81ce9fe0442028d6abeb26ad1c554 is 50, key is test_row_0/B:col10/1731575730921/Put/seqid=0 2024-11-14T09:15:33,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742209_1385 (size=12151) 2024-11-14T09:15:34,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-14T09:15:34,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-14T09:15:34,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:34,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. as already flushing 2024-11-14T09:15:34,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:34,061 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:34,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:34,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:34,213 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-14T09:15:34,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:34,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. as already flushing 2024-11-14T09:15:34,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:34,214 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:34,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:34,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:34,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-14T09:15:34,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:34,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:34,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56516 deadline: 1731575794344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:34,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56618 deadline: 1731575794344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:34,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:34,356 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:34,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:34,356 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:34,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:34,356 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:34,366 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-14T09:15:34,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:34,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. as already flushing 2024-11-14T09:15:34,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:34,366 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:34,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:34,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:34,386 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/B/e8d81ce9fe0442028d6abeb26ad1c554 2024-11-14T09:15:34,393 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/C/91e325f61998490bba77fa647b708739 is 50, key is test_row_0/C:col10/1731575730921/Put/seqid=0 2024-11-14T09:15:34,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742210_1386 (size=12151) 2024-11-14T09:15:34,519 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-14T09:15:34,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:34,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. as already flushing 2024-11-14T09:15:34,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:34,519 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:34,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:34,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:34,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-14T09:15:34,672 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-14T09:15:34,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:34,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. as already flushing 2024-11-14T09:15:34,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:34,672 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:34,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:34,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:34,797 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/C/91e325f61998490bba77fa647b708739 2024-11-14T09:15:34,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/13ffeb1b07834f8797a073fa82ae7e30 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/13ffeb1b07834f8797a073fa82ae7e30 2024-11-14T09:15:34,804 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/13ffeb1b07834f8797a073fa82ae7e30, entries=200, sequenceid=194, filesize=38.8 K 2024-11-14T09:15:34,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/B/e8d81ce9fe0442028d6abeb26ad1c554 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/e8d81ce9fe0442028d6abeb26ad1c554 2024-11-14T09:15:34,807 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/e8d81ce9fe0442028d6abeb26ad1c554, entries=150, sequenceid=194, filesize=11.9 K 2024-11-14T09:15:34,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/C/91e325f61998490bba77fa647b708739 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/91e325f61998490bba77fa647b708739 2024-11-14T09:15:34,811 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/91e325f61998490bba77fa647b708739, entries=150, sequenceid=194, filesize=11.9 K 2024-11-14T09:15:34,812 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for 3e7b42f1efdfdf4879c3f173e59339f9 in 1668ms, sequenceid=194, compaction requested=true 2024-11-14T09:15:34,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3e7b42f1efdfdf4879c3f173e59339f9: 2024-11-14T09:15:34,812 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:15:34,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3e7b42f1efdfdf4879c3f173e59339f9:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:15:34,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:15:34,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3e7b42f1efdfdf4879c3f173e59339f9:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:15:34,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:15:34,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3e7b42f1efdfdf4879c3f173e59339f9:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:15:34,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:15:34,813 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:15:34,813 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102165 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:15:34,813 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): 3e7b42f1efdfdf4879c3f173e59339f9/A is initiating minor compaction (all files) 2024-11-14T09:15:34,814 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3e7b42f1efdfdf4879c3f173e59339f9/A in TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:34,814 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/d649935043724b438195a8817ff9868c, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/fc638de742f64c3e9eef03fe047e63df, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/13ffeb1b07834f8797a073fa82ae7e30] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp, totalSize=99.8 K 2024-11-14T09:15:34,814 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:34,814 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. files: [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/d649935043724b438195a8817ff9868c, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/fc638de742f64c3e9eef03fe047e63df, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/13ffeb1b07834f8797a073fa82ae7e30] 2024-11-14T09:15:34,814 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:15:34,814 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): 3e7b42f1efdfdf4879c3f173e59339f9/B is initiating minor compaction (all files) 2024-11-14T09:15:34,814 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3e7b42f1efdfdf4879c3f173e59339f9/B in TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:34,814 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/0e48131d01d143eca9996e938a879208, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/3314466ada124ff0a1d3c7b7b8f70870, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/e8d81ce9fe0442028d6abeb26ad1c554] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp, totalSize=35.8 K 2024-11-14T09:15:34,815 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e48131d01d143eca9996e938a879208, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731575728348 2024-11-14T09:15:34,815 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting d649935043724b438195a8817ff9868c, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731575728348 2024-11-14T09:15:34,815 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc638de742f64c3e9eef03fe047e63df, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731575729591 2024-11-14T09:15:34,815 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 3314466ada124ff0a1d3c7b7b8f70870, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731575729591 2024-11-14T09:15:34,815 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13ffeb1b07834f8797a073fa82ae7e30, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731575730911 2024-11-14T09:15:34,815 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting e8d81ce9fe0442028d6abeb26ad1c554, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731575730911 2024-11-14T09:15:34,821 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3e7b42f1efdfdf4879c3f173e59339f9] 2024-11-14T09:15:34,823 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411142deb8b7540ad41fa8afbc597c7a481db_3e7b42f1efdfdf4879c3f173e59339f9 store=[table=TestAcidGuarantees family=A region=3e7b42f1efdfdf4879c3f173e59339f9] 2024-11-14T09:15:34,823 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3e7b42f1efdfdf4879c3f173e59339f9#B#compaction#328 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:15:34,824 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/B/9e207e38bd90454387ef9941896740a8 is 50, key is test_row_0/B:col10/1731575730921/Put/seqid=0 2024-11-14T09:15:34,825 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411142deb8b7540ad41fa8afbc597c7a481db_3e7b42f1efdfdf4879c3f173e59339f9, store=[table=TestAcidGuarantees family=A region=3e7b42f1efdfdf4879c3f173e59339f9] 2024-11-14T09:15:34,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-14T09:15:34,825 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411142deb8b7540ad41fa8afbc597c7a481db_3e7b42f1efdfdf4879c3f173e59339f9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3e7b42f1efdfdf4879c3f173e59339f9] 2024-11-14T09:15:34,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:34,825 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 3e7b42f1efdfdf4879c3f173e59339f9 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-14T09:15:34,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e7b42f1efdfdf4879c3f173e59339f9, store=A 2024-11-14T09:15:34,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:15:34,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e7b42f1efdfdf4879c3f173e59339f9, store=B 2024-11-14T09:15:34,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:15:34,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e7b42f1efdfdf4879c3f173e59339f9, store=C 2024-11-14T09:15:34,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:15:34,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742211_1387 (size=12459) 2024-11-14T09:15:34,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114600b3eec90cd4763aed0795b0525fe09_3e7b42f1efdfdf4879c3f173e59339f9 is 50, key is test_row_0/A:col10/1731575733157/Put/seqid=0 2024-11-14T09:15:34,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742212_1388 (size=4469) 2024-11-14T09:15:34,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742213_1389 (size=12304) 2024-11-14T09:15:35,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-14T09:15:35,238 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/B/9e207e38bd90454387ef9941896740a8 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/9e207e38bd90454387ef9941896740a8 2024-11-14T09:15:35,242 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3e7b42f1efdfdf4879c3f173e59339f9/B of 3e7b42f1efdfdf4879c3f173e59339f9 into 9e207e38bd90454387ef9941896740a8(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:15:35,242 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3e7b42f1efdfdf4879c3f173e59339f9: 2024-11-14T09:15:35,242 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., storeName=3e7b42f1efdfdf4879c3f173e59339f9/B, priority=13, startTime=1731575734813; duration=0sec 2024-11-14T09:15:35,242 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:15:35,242 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3e7b42f1efdfdf4879c3f173e59339f9:B 2024-11-14T09:15:35,242 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:15:35,243 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:15:35,243 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): 3e7b42f1efdfdf4879c3f173e59339f9/C is initiating minor compaction (all files) 2024-11-14T09:15:35,243 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3e7b42f1efdfdf4879c3f173e59339f9/C in TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:35,243 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/d76752d352984488bea5d6725bc1619f, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/6a3e9821aae74556a4ee90fa639965f2, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/91e325f61998490bba77fa647b708739] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp, totalSize=35.8 K 2024-11-14T09:15:35,243 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting d76752d352984488bea5d6725bc1619f, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731575728348 2024-11-14T09:15:35,244 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a3e9821aae74556a4ee90fa639965f2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731575729591 2024-11-14T09:15:35,244 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 91e325f61998490bba77fa647b708739, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731575730911 2024-11-14T09:15:35,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:15:35,257 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3e7b42f1efdfdf4879c3f173e59339f9#A#compaction#327 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:15:35,258 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/e6b13cdd5f054d63a0dc4f4a9f2f5073 is 175, key is test_row_0/A:col10/1731575730921/Put/seqid=0 2024-11-14T09:15:35,262 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114600b3eec90cd4763aed0795b0525fe09_3e7b42f1efdfdf4879c3f173e59339f9 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114600b3eec90cd4763aed0795b0525fe09_3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:35,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/d30bb31b4679475a97ca5bf3958cf6cf, store: [table=TestAcidGuarantees family=A region=3e7b42f1efdfdf4879c3f173e59339f9] 2024-11-14T09:15:35,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/d30bb31b4679475a97ca5bf3958cf6cf is 175, key is test_row_0/A:col10/1731575733157/Put/seqid=0 2024-11-14T09:15:35,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742214_1390 (size=31413) 2024-11-14T09:15:35,267 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3e7b42f1efdfdf4879c3f173e59339f9#C#compaction#330 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:15:35,267 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/C/8da9179cc0fc4f80ae03d01d58345674 is 50, key is test_row_0/C:col10/1731575730921/Put/seqid=0 2024-11-14T09:15:35,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742215_1391 (size=31105) 2024-11-14T09:15:35,273 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=203, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/d30bb31b4679475a97ca5bf3958cf6cf 2024-11-14T09:15:35,273 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/e6b13cdd5f054d63a0dc4f4a9f2f5073 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/e6b13cdd5f054d63a0dc4f4a9f2f5073 2024-11-14T09:15:35,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742216_1392 (size=12459) 2024-11-14T09:15:35,283 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/C/8da9179cc0fc4f80ae03d01d58345674 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/8da9179cc0fc4f80ae03d01d58345674 2024-11-14T09:15:35,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/B/641b258c7ad244f29a0c22ce3d53bd36 is 50, key is test_row_0/B:col10/1731575733157/Put/seqid=0 2024-11-14T09:15:35,288 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3e7b42f1efdfdf4879c3f173e59339f9/C of 3e7b42f1efdfdf4879c3f173e59339f9 into 8da9179cc0fc4f80ae03d01d58345674(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:15:35,288 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3e7b42f1efdfdf4879c3f173e59339f9: 2024-11-14T09:15:35,288 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., storeName=3e7b42f1efdfdf4879c3f173e59339f9/C, priority=13, startTime=1731575734813; duration=0sec 2024-11-14T09:15:35,288 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:15:35,288 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3e7b42f1efdfdf4879c3f173e59339f9:C 2024-11-14T09:15:35,289 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3e7b42f1efdfdf4879c3f173e59339f9/A of 3e7b42f1efdfdf4879c3f173e59339f9 into e6b13cdd5f054d63a0dc4f4a9f2f5073(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:15:35,289 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3e7b42f1efdfdf4879c3f173e59339f9: 2024-11-14T09:15:35,289 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., storeName=3e7b42f1efdfdf4879c3f173e59339f9/A, priority=13, startTime=1731575734812; duration=0sec 2024-11-14T09:15:35,289 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:15:35,289 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3e7b42f1efdfdf4879c3f173e59339f9:A 2024-11-14T09:15:35,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742217_1393 (size=12151) 2024-11-14T09:15:35,290 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/B/641b258c7ad244f29a0c22ce3d53bd36 2024-11-14T09:15:35,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/C/54867843ea2547f79b47ca63461c7529 is 50, key is test_row_0/C:col10/1731575733157/Put/seqid=0 2024-11-14T09:15:35,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742218_1394 (size=12151) 2024-11-14T09:15:35,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on 3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:35,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. as already flushing 2024-11-14T09:15:35,440 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731575735398, max timestamp=1731575735438Num uniques 859; 2024-11-14T09:15:35,441 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T09:15:35,441 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731575735398, max timestamp=1731575735438Num uniques 859; 2024-11-14T09:15:35,441 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T09:15:35,441 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731575735398, max timestamp=1731575735438Num uniques 859; 2024-11-14T09:15:35,441 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T09:15:35,541 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731575735469, max timestamp=1731575735523Num uniques 1068; 2024-11-14T09:15:35,541 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-14T09:15:35,541 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731575735469, max timestamp=1731575735523Num uniques 1068; 2024-11-14T09:15:35,541 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-14T09:15:35,541 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731575735469, max timestamp=1731575735523Num uniques 1068; 2024-11-14T09:15:35,541 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-14T09:15:35,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:35,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56618 deadline: 1731575795611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:35,623 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:35,624 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:35,624 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:35,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:35,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56516 deadline: 1731575795617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:35,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:35,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:35,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:35,718 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/C/54867843ea2547f79b47ca63461c7529 2024-11-14T09:15:35,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/d30bb31b4679475a97ca5bf3958cf6cf as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/d30bb31b4679475a97ca5bf3958cf6cf 2024-11-14T09:15:35,729 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/d30bb31b4679475a97ca5bf3958cf6cf, entries=150, sequenceid=203, filesize=30.4 K 2024-11-14T09:15:35,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/B/641b258c7ad244f29a0c22ce3d53bd36 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/641b258c7ad244f29a0c22ce3d53bd36 2024-11-14T09:15:35,733 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/641b258c7ad244f29a0c22ce3d53bd36, entries=150, sequenceid=203, filesize=11.9 K 2024-11-14T09:15:35,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/C/54867843ea2547f79b47ca63461c7529 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/54867843ea2547f79b47ca63461c7529 2024-11-14T09:15:35,738 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/54867843ea2547f79b47ca63461c7529, entries=150, sequenceid=203, filesize=11.9 K 2024-11-14T09:15:35,738 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 3e7b42f1efdfdf4879c3f173e59339f9 in 913ms, sequenceid=203, compaction requested=false 2024-11-14T09:15:35,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 3e7b42f1efdfdf4879c3f173e59339f9: 2024-11-14T09:15:35,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:35,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-14T09:15:35,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-14T09:15:35,741 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-14T09:15:35,741 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8320 sec 2024-11-14T09:15:35,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on 3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:35,742 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3e7b42f1efdfdf4879c3f173e59339f9 3/3 column families, dataSize=147.60 KB heapSize=420 KB 2024-11-14T09:15:35,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e7b42f1efdfdf4879c3f173e59339f9, store=A 2024-11-14T09:15:35,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-14T09:15:35,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e7b42f1efdfdf4879c3f173e59339f9, store=B 2024-11-14T09:15:35,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-14T09:15:35,744 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.8360 sec 2024-11-14T09:15:35,746 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731575735572, max timestamp=1731575735741Num uniques 1077; 2024-11-14T09:15:35,746 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-14T09:15:35,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e7b42f1efdfdf4879c3f173e59339f9, store=C 2024-11-14T09:15:35,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:15:35,758 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411143c865bd73a424cefbb8e11a9ecab003d_3e7b42f1efdfdf4879c3f173e59339f9 is 50, key is test_row_0/A:col10/1731575735572/Put/seqid=0 2024-11-14T09:15:35,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742219_1395 (size=14894) 2024-11-14T09:15:35,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:35,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56516 deadline: 1731575795829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:35,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:35,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:35,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:35,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:35,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56618 deadline: 1731575795840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:35,842 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:35,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:35,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:35,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:35,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56516 deadline: 1731575795945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:35,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:35,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:35,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:35,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56618 deadline: 1731575795945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:35,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:35,958 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:35,958 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:35,958 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:36,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-14T09:15:36,056 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:15:36,056 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:15:36,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-14T09:15:36,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-14T09:15:36,058 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:15:36,059 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:15:36,059 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:15:36,162 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:15:36,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-14T09:15:36,166 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411143c865bd73a424cefbb8e11a9ecab003d_3e7b42f1efdfdf4879c3f173e59339f9 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143c865bd73a424cefbb8e11a9ecab003d_3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:36,167 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/1195d60340f64a19b2502a6ed04a5a8e, store: [table=TestAcidGuarantees family=A region=3e7b42f1efdfdf4879c3f173e59339f9] 2024-11-14T09:15:36,168 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/1195d60340f64a19b2502a6ed04a5a8e is 175, key is test_row_0/A:col10/1731575735572/Put/seqid=0 2024-11-14T09:15:36,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:36,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56516 deadline: 1731575796165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:36,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:36,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:36,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:36,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56618 deadline: 1731575796165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:36,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:36,170 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:36,170 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:36,170 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:36,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742220_1396 (size=39849) 2024-11-14T09:15:36,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-14T09:15:36,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:36,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. as already flushing 2024-11-14T09:15:36,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:36,211 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:36,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:36,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:36,364 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-14T09:15:36,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:36,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. as already flushing 2024-11-14T09:15:36,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:36,364 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:36,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:36,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:36,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-14T09:15:36,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:36,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56516 deadline: 1731575796475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:36,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:36,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:36,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:36,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:36,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56618 deadline: 1731575796475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:36,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:36,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:36,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:36,517 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-14T09:15:36,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:36,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. as already flushing 2024-11-14T09:15:36,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:36,517 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:36,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:36,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:36,574 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=261, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/1195d60340f64a19b2502a6ed04a5a8e 2024-11-14T09:15:36,582 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/B/8a8237991ccd4fe7a44e8b66ad7b3d23 is 50, key is test_row_0/B:col10/1731575735572/Put/seqid=0 2024-11-14T09:15:36,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742221_1397 (size=12251) 2024-11-14T09:15:36,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-14T09:15:36,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:36,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. as already flushing 2024-11-14T09:15:36,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:36,670 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:36,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:36,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:36,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-14T09:15:36,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-14T09:15:36,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:36,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. as already flushing 2024-11-14T09:15:36,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:36,823 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:36,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:36,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:36,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-14T09:15:36,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:36,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. as already flushing 2024-11-14T09:15:36,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:36,977 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:36,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:36,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:36,986 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/B/8a8237991ccd4fe7a44e8b66ad7b3d23 2024-11-14T09:15:36,993 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/C/4c67b3a114f546e79ab39e0ad87576d1 is 50, key is test_row_0/C:col10/1731575735572/Put/seqid=0 2024-11-14T09:15:36,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742222_1398 (size=12251) 2024-11-14T09:15:37,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:37,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56516 deadline: 1731575796995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:37,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:37,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:37,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:37,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:37,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56618 deadline: 1731575796995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:37,004 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:37,004 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:37,004 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:37,130 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-14T09:15:37,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:37,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. as already flushing 2024-11-14T09:15:37,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:37,130 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:37,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:37,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:37,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-14T09:15:37,283 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-14T09:15:37,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:37,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. as already flushing 2024-11-14T09:15:37,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:37,283 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:37,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:37,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:37,397 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/C/4c67b3a114f546e79ab39e0ad87576d1 2024-11-14T09:15:37,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/1195d60340f64a19b2502a6ed04a5a8e as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/1195d60340f64a19b2502a6ed04a5a8e 2024-11-14T09:15:37,404 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/1195d60340f64a19b2502a6ed04a5a8e, entries=200, sequenceid=261, filesize=38.9 K 2024-11-14T09:15:37,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/B/8a8237991ccd4fe7a44e8b66ad7b3d23 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/8a8237991ccd4fe7a44e8b66ad7b3d23 2024-11-14T09:15:37,408 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/8a8237991ccd4fe7a44e8b66ad7b3d23, entries=150, sequenceid=261, filesize=12.0 K 2024-11-14T09:15:37,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/C/4c67b3a114f546e79ab39e0ad87576d1 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/4c67b3a114f546e79ab39e0ad87576d1 2024-11-14T09:15:37,412 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/4c67b3a114f546e79ab39e0ad87576d1, entries=150, sequenceid=261, filesize=12.0 K 2024-11-14T09:15:37,413 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~301.47 KB/308704, currentSize=73.80 KB/75570 for 3e7b42f1efdfdf4879c3f173e59339f9 in 1671ms, sequenceid=261, compaction requested=true 2024-11-14T09:15:37,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3e7b42f1efdfdf4879c3f173e59339f9: 2024-11-14T09:15:37,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3e7b42f1efdfdf4879c3f173e59339f9:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:15:37,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:15:37,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3e7b42f1efdfdf4879c3f173e59339f9:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:15:37,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T09:15:37,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3e7b42f1efdfdf4879c3f173e59339f9:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:15:37,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-14T09:15:37,414 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:15:37,414 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:15:37,415 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36861 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:15:37,415 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): 3e7b42f1efdfdf4879c3f173e59339f9/C is initiating minor compaction (all files) 2024-11-14T09:15:37,415 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3e7b42f1efdfdf4879c3f173e59339f9/C in TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:37,415 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/8da9179cc0fc4f80ae03d01d58345674, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/54867843ea2547f79b47ca63461c7529, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/4c67b3a114f546e79ab39e0ad87576d1] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp, totalSize=36.0 K 2024-11-14T09:15:37,416 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 8da9179cc0fc4f80ae03d01d58345674, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731575730911 2024-11-14T09:15:37,416 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:15:37,416 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): 3e7b42f1efdfdf4879c3f173e59339f9/A is initiating minor compaction (all files) 2024-11-14T09:15:37,416 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3e7b42f1efdfdf4879c3f173e59339f9/A in TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:37,416 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/e6b13cdd5f054d63a0dc4f4a9f2f5073, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/d30bb31b4679475a97ca5bf3958cf6cf, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/1195d60340f64a19b2502a6ed04a5a8e] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp, totalSize=100.0 K 2024-11-14T09:15:37,416 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:37,416 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. files: [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/e6b13cdd5f054d63a0dc4f4a9f2f5073, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/d30bb31b4679475a97ca5bf3958cf6cf, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/1195d60340f64a19b2502a6ed04a5a8e] 2024-11-14T09:15:37,417 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 54867843ea2547f79b47ca63461c7529, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731575733157 2024-11-14T09:15:37,417 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6b13cdd5f054d63a0dc4f4a9f2f5073, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731575730911 2024-11-14T09:15:37,417 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c67b3a114f546e79ab39e0ad87576d1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731575735572 2024-11-14T09:15:37,417 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting d30bb31b4679475a97ca5bf3958cf6cf, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731575733157 2024-11-14T09:15:37,418 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1195d60340f64a19b2502a6ed04a5a8e, keycount=200, bloomtype=ROW, size=38.9 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731575735572 2024-11-14T09:15:37,428 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3e7b42f1efdfdf4879c3f173e59339f9#C#compaction#336 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:15:37,429 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/C/0459f6b0fea44368a888bbdbb7d54c94 is 50, key is test_row_0/C:col10/1731575735572/Put/seqid=0 2024-11-14T09:15:37,430 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3e7b42f1efdfdf4879c3f173e59339f9] 2024-11-14T09:15:37,432 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114323461634b3c4152aa259b7d2b72c9bb_3e7b42f1efdfdf4879c3f173e59339f9 store=[table=TestAcidGuarantees family=A region=3e7b42f1efdfdf4879c3f173e59339f9] 2024-11-14T09:15:37,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742223_1399 (size=12661) 2024-11-14T09:15:37,434 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114323461634b3c4152aa259b7d2b72c9bb_3e7b42f1efdfdf4879c3f173e59339f9, store=[table=TestAcidGuarantees family=A region=3e7b42f1efdfdf4879c3f173e59339f9] 2024-11-14T09:15:37,434 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114323461634b3c4152aa259b7d2b72c9bb_3e7b42f1efdfdf4879c3f173e59339f9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3e7b42f1efdfdf4879c3f173e59339f9] 2024-11-14T09:15:37,436 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-14T09:15:37,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:37,436 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 3e7b42f1efdfdf4879c3f173e59339f9 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-14T09:15:37,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e7b42f1efdfdf4879c3f173e59339f9, store=A 2024-11-14T09:15:37,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:15:37,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e7b42f1efdfdf4879c3f173e59339f9, store=B 2024-11-14T09:15:37,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:15:37,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e7b42f1efdfdf4879c3f173e59339f9, store=C 2024-11-14T09:15:37,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:15:37,441 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/C/0459f6b0fea44368a888bbdbb7d54c94 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/0459f6b0fea44368a888bbdbb7d54c94 2024-11-14T09:15:37,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111489ff447c450e44d888d2df5bd3610388_3e7b42f1efdfdf4879c3f173e59339f9 is 50, key is test_row_0/A:col10/1731575735839/Put/seqid=0 2024-11-14T09:15:37,448 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3e7b42f1efdfdf4879c3f173e59339f9/C of 3e7b42f1efdfdf4879c3f173e59339f9 into 0459f6b0fea44368a888bbdbb7d54c94(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:15:37,448 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3e7b42f1efdfdf4879c3f173e59339f9: 2024-11-14T09:15:37,448 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., storeName=3e7b42f1efdfdf4879c3f173e59339f9/C, priority=13, startTime=1731575737414; duration=0sec 2024-11-14T09:15:37,448 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:15:37,448 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3e7b42f1efdfdf4879c3f173e59339f9:C 2024-11-14T09:15:37,448 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:15:37,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742224_1400 (size=4469) 2024-11-14T09:15:37,449 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36861 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:15:37,450 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): 3e7b42f1efdfdf4879c3f173e59339f9/B is initiating minor compaction (all files) 2024-11-14T09:15:37,450 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3e7b42f1efdfdf4879c3f173e59339f9/B in TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:37,450 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/9e207e38bd90454387ef9941896740a8, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/641b258c7ad244f29a0c22ce3d53bd36, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/8a8237991ccd4fe7a44e8b66ad7b3d23] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp, totalSize=36.0 K 2024-11-14T09:15:37,450 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e207e38bd90454387ef9941896740a8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731575730911 2024-11-14T09:15:37,450 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3e7b42f1efdfdf4879c3f173e59339f9#A#compaction#337 average throughput is 1.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:15:37,450 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 641b258c7ad244f29a0c22ce3d53bd36, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731575733157 2024-11-14T09:15:37,451 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a8237991ccd4fe7a44e8b66ad7b3d23, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731575735572 2024-11-14T09:15:37,451 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/93c3a59eb8d64eea9b3653b6ad5136e0 is 175, key is test_row_0/A:col10/1731575735572/Put/seqid=0 2024-11-14T09:15:37,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742225_1401 (size=12454) 2024-11-14T09:15:37,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:15:37,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742226_1402 (size=31615) 2024-11-14T09:15:37,464 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111489ff447c450e44d888d2df5bd3610388_3e7b42f1efdfdf4879c3f173e59339f9 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111489ff447c450e44d888d2df5bd3610388_3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:37,466 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3e7b42f1efdfdf4879c3f173e59339f9#B#compaction#339 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:15:37,467 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/B/057521bf67874bcda8f04ccc324e832d is 50, key is test_row_0/B:col10/1731575735572/Put/seqid=0 2024-11-14T09:15:37,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/3ad25bbfbd634429a9188833c9bde061, store: [table=TestAcidGuarantees family=A region=3e7b42f1efdfdf4879c3f173e59339f9] 2024-11-14T09:15:37,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/3ad25bbfbd634429a9188833c9bde061 is 175, key is test_row_0/A:col10/1731575735839/Put/seqid=0 2024-11-14T09:15:37,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742228_1404 (size=31255) 2024-11-14T09:15:37,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742227_1403 (size=12661) 2024-11-14T09:15:37,868 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/93c3a59eb8d64eea9b3653b6ad5136e0 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/93c3a59eb8d64eea9b3653b6ad5136e0 2024-11-14T09:15:37,872 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3e7b42f1efdfdf4879c3f173e59339f9/A of 3e7b42f1efdfdf4879c3f173e59339f9 into 93c3a59eb8d64eea9b3653b6ad5136e0(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:15:37,872 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3e7b42f1efdfdf4879c3f173e59339f9: 2024-11-14T09:15:37,872 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., storeName=3e7b42f1efdfdf4879c3f173e59339f9/A, priority=13, startTime=1731575737413; duration=0sec 2024-11-14T09:15:37,872 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:15:37,872 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3e7b42f1efdfdf4879c3f173e59339f9:A 2024-11-14T09:15:37,879 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=275, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/3ad25bbfbd634429a9188833c9bde061 2024-11-14T09:15:37,883 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/B/057521bf67874bcda8f04ccc324e832d as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/057521bf67874bcda8f04ccc324e832d 2024-11-14T09:15:37,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/B/3bc1ff2c26a745dbbce659947d90cff5 is 50, key is test_row_0/B:col10/1731575735839/Put/seqid=0 2024-11-14T09:15:37,888 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3e7b42f1efdfdf4879c3f173e59339f9/B of 3e7b42f1efdfdf4879c3f173e59339f9 into 057521bf67874bcda8f04ccc324e832d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:15:37,888 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3e7b42f1efdfdf4879c3f173e59339f9: 2024-11-14T09:15:37,888 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., storeName=3e7b42f1efdfdf4879c3f173e59339f9/B, priority=13, startTime=1731575737414; duration=0sec 2024-11-14T09:15:37,888 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:15:37,888 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3e7b42f1efdfdf4879c3f173e59339f9:B 2024-11-14T09:15:37,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742229_1405 (size=12301) 2024-11-14T09:15:38,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. as already flushing 2024-11-14T09:15:38,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on 3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:38,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:38,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56516 deadline: 1731575798075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:38,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:38,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:38,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:38,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:38,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56618 deadline: 1731575798076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:38,088 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:38,088 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:38,088 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:38,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:38,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56516 deadline: 1731575798185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:38,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:38,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:38,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:38,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:38,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56618 deadline: 1731575798195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:38,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:38,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:38,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:38,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-14T09:15:38,295 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/B/3bc1ff2c26a745dbbce659947d90cff5 2024-11-14T09:15:38,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/C/fdb90e36398046a08fe1e483706ee02c is 50, key is test_row_0/C:col10/1731575735839/Put/seqid=0 2024-11-14T09:15:38,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742230_1406 (size=12301) 2024-11-14T09:15:38,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:38,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56516 deadline: 1731575798395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:38,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:38,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:38,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:38,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:38,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56618 deadline: 1731575798415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:38,419 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:38,419 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:38,419 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:38,705 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/C/fdb90e36398046a08fe1e483706ee02c 2024-11-14T09:15:38,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/3ad25bbfbd634429a9188833c9bde061 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/3ad25bbfbd634429a9188833c9bde061 2024-11-14T09:15:38,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:38,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56516 deadline: 1731575798705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:38,711 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:38,711 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:38,711 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:38,713 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/3ad25bbfbd634429a9188833c9bde061, entries=150, sequenceid=275, filesize=30.5 K 2024-11-14T09:15:38,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/B/3bc1ff2c26a745dbbce659947d90cff5 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/3bc1ff2c26a745dbbce659947d90cff5 2024-11-14T09:15:38,716 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/3bc1ff2c26a745dbbce659947d90cff5, entries=150, sequenceid=275, filesize=12.0 K 2024-11-14T09:15:38,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/C/fdb90e36398046a08fe1e483706ee02c as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/fdb90e36398046a08fe1e483706ee02c 2024-11-14T09:15:38,721 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/fdb90e36398046a08fe1e483706ee02c, entries=150, sequenceid=275, filesize=12.0 K 2024-11-14T09:15:38,721 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 3e7b42f1efdfdf4879c3f173e59339f9 in 1285ms, sequenceid=275, compaction requested=false 2024-11-14T09:15:38,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 3e7b42f1efdfdf4879c3f173e59339f9: 2024-11-14T09:15:38,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:38,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-14T09:15:38,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-14T09:15:38,725 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-14T09:15:38,725 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6640 sec 2024-11-14T09:15:38,727 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.6690 sec 2024-11-14T09:15:38,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on 3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:38,728 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3e7b42f1efdfdf4879c3f173e59339f9 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-14T09:15:38,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e7b42f1efdfdf4879c3f173e59339f9, store=A 2024-11-14T09:15:38,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:15:38,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e7b42f1efdfdf4879c3f173e59339f9, store=B 2024-11-14T09:15:38,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:15:38,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e7b42f1efdfdf4879c3f173e59339f9, store=C 2024-11-14T09:15:38,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:15:38,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411148bbeb9c2c4f74e0d936ef7161147dbae_3e7b42f1efdfdf4879c3f173e59339f9 is 50, key is test_row_0/A:col10/1731575738727/Put/seqid=0 2024-11-14T09:15:38,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742231_1407 (size=14994) 2024-11-14T09:15:38,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:38,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56618 deadline: 1731575798816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:38,831 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:38,831 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:38,831 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:38,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:38,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56618 deadline: 1731575798935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:38,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:38,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:38,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:39,143 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:15:39,146 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411148bbeb9c2c4f74e0d936ef7161147dbae_3e7b42f1efdfdf4879c3f173e59339f9 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411148bbeb9c2c4f74e0d936ef7161147dbae_3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:39,147 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/917236fcda7e4e7e84941e448ac8f55d, store: [table=TestAcidGuarantees family=A region=3e7b42f1efdfdf4879c3f173e59339f9] 2024-11-14T09:15:39,148 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/917236fcda7e4e7e84941e448ac8f55d is 175, key is test_row_0/A:col10/1731575738727/Put/seqid=0 2024-11-14T09:15:39,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742232_1408 (size=39949) 2024-11-14T09:15:39,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:39,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56618 deadline: 1731575799155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:39,162 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:39,162 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:39,162 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:39,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:39,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56516 deadline: 1731575799224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:39,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:39,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:39,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:39,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:39,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56618 deadline: 1731575799465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:39,473 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:39,473 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:39,473 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:39,552 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=297, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/917236fcda7e4e7e84941e448ac8f55d 2024-11-14T09:15:39,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/B/3601016207ee4ad9b21822e1e64af55d is 50, key is test_row_0/B:col10/1731575738727/Put/seqid=0 2024-11-14T09:15:39,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742233_1409 (size=12301) 2024-11-14T09:15:39,963 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/B/3601016207ee4ad9b21822e1e64af55d 2024-11-14T09:15:39,970 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/C/8a86429a1f9f41d3b865de240ffeca93 is 50, key is test_row_0/C:col10/1731575738727/Put/seqid=0 2024-11-14T09:15:39,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742234_1410 (size=12301) 2024-11-14T09:15:39,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:39,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56618 deadline: 1731575799985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:39,998 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:39,998 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:39,998 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:40,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-14T09:15:40,225 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:15:40,226 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:15:40,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-14T09:15:40,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-14T09:15:40,227 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:15:40,228 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:15:40,228 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:15:40,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:40,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56516 deadline: 1731575800235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:40,240 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_0' in 3e7b42f1efdfdf4879c3f173e59339f9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:15:40,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:40,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:40,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:40,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-14T09:15:40,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:40,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56556 deadline: 1731575800365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:40,372 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_0' in 3e7b42f1efdfdf4879c3f173e59339f9 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18296 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:15:40,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:40,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:40,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:40,374 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/C/8a86429a1f9f41d3b865de240ffeca93 2024-11-14T09:15:40,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/917236fcda7e4e7e84941e448ac8f55d as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/917236fcda7e4e7e84941e448ac8f55d 2024-11-14T09:15:40,381 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-14T09:15:40,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:40,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. as already flushing 2024-11-14T09:15:40,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:40,381 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:40,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:40,382 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/917236fcda7e4e7e84941e448ac8f55d, entries=200, sequenceid=297, filesize=39.0 K 2024-11-14T09:15:40,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:40,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/B/3601016207ee4ad9b21822e1e64af55d as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/3601016207ee4ad9b21822e1e64af55d 2024-11-14T09:15:40,387 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/3601016207ee4ad9b21822e1e64af55d, entries=150, sequenceid=297, filesize=12.0 K 2024-11-14T09:15:40,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/C/8a86429a1f9f41d3b865de240ffeca93 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/8a86429a1f9f41d3b865de240ffeca93 2024-11-14T09:15:40,391 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/8a86429a1f9f41d3b865de240ffeca93, entries=150, sequenceid=297, filesize=12.0 K 2024-11-14T09:15:40,391 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 3e7b42f1efdfdf4879c3f173e59339f9 in 1663ms, sequenceid=297, compaction requested=true 2024-11-14T09:15:40,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3e7b42f1efdfdf4879c3f173e59339f9: 2024-11-14T09:15:40,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3e7b42f1efdfdf4879c3f173e59339f9:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:15:40,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:15:40,392 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:15:40,392 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:15:40,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3e7b42f1efdfdf4879c3f173e59339f9:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:15:40,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:15:40,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3e7b42f1efdfdf4879c3f173e59339f9:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:15:40,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:15:40,393 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102819 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:15:40,393 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37263 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:15:40,393 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): 3e7b42f1efdfdf4879c3f173e59339f9/A is initiating minor compaction (all files) 2024-11-14T09:15:40,393 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): 3e7b42f1efdfdf4879c3f173e59339f9/B is initiating minor compaction (all files) 2024-11-14T09:15:40,393 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3e7b42f1efdfdf4879c3f173e59339f9/B in TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:40,393 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3e7b42f1efdfdf4879c3f173e59339f9/A in TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:40,393 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/93c3a59eb8d64eea9b3653b6ad5136e0, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/3ad25bbfbd634429a9188833c9bde061, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/917236fcda7e4e7e84941e448ac8f55d] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp, totalSize=100.4 K 2024-11-14T09:15:40,393 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/057521bf67874bcda8f04ccc324e832d, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/3bc1ff2c26a745dbbce659947d90cff5, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/3601016207ee4ad9b21822e1e64af55d] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp, totalSize=36.4 K 2024-11-14T09:15:40,393 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:40,393 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. files: [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/93c3a59eb8d64eea9b3653b6ad5136e0, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/3ad25bbfbd634429a9188833c9bde061, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/917236fcda7e4e7e84941e448ac8f55d] 2024-11-14T09:15:40,393 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 057521bf67874bcda8f04ccc324e832d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731575735572 2024-11-14T09:15:40,393 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93c3a59eb8d64eea9b3653b6ad5136e0, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731575735572 2024-11-14T09:15:40,394 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bc1ff2c26a745dbbce659947d90cff5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731575735808 2024-11-14T09:15:40,394 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ad25bbfbd634429a9188833c9bde061, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731575735808 2024-11-14T09:15:40,394 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 3601016207ee4ad9b21822e1e64af55d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731575738033 2024-11-14T09:15:40,394 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 917236fcda7e4e7e84941e448ac8f55d, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731575738033 2024-11-14T09:15:40,400 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3e7b42f1efdfdf4879c3f173e59339f9] 2024-11-14T09:15:40,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on 3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:40,401 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3e7b42f1efdfdf4879c3f173e59339f9 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-14T09:15:40,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e7b42f1efdfdf4879c3f173e59339f9, store=A 2024-11-14T09:15:40,401 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3e7b42f1efdfdf4879c3f173e59339f9#B#compaction#345 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:15:40,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:15:40,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e7b42f1efdfdf4879c3f173e59339f9, store=B 2024-11-14T09:15:40,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:15:40,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e7b42f1efdfdf4879c3f173e59339f9, store=C 2024-11-14T09:15:40,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:15:40,402 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/B/b3c38347e4a04cc4841fbc0ecaea2d84 is 50, key is test_row_0/B:col10/1731575738727/Put/seqid=0 2024-11-14T09:15:40,404 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114f80631495e7f4a8f85f62b0eba62af45_3e7b42f1efdfdf4879c3f173e59339f9 store=[table=TestAcidGuarantees family=A region=3e7b42f1efdfdf4879c3f173e59339f9] 2024-11-14T09:15:40,407 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114f80631495e7f4a8f85f62b0eba62af45_3e7b42f1efdfdf4879c3f173e59339f9, store=[table=TestAcidGuarantees family=A region=3e7b42f1efdfdf4879c3f173e59339f9] 2024-11-14T09:15:40,407 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114f80631495e7f4a8f85f62b0eba62af45_3e7b42f1efdfdf4879c3f173e59339f9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3e7b42f1efdfdf4879c3f173e59339f9] 2024-11-14T09:15:40,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742235_1411 (size=12813) 2024-11-14T09:15:40,420 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114aa8a6af015d9436dbcea53e38ddce548_3e7b42f1efdfdf4879c3f173e59339f9 is 50, key is test_row_0/A:col10/1731575738809/Put/seqid=0 2024-11-14T09:15:40,422 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/B/b3c38347e4a04cc4841fbc0ecaea2d84 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/b3c38347e4a04cc4841fbc0ecaea2d84 2024-11-14T09:15:40,427 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3e7b42f1efdfdf4879c3f173e59339f9/B of 3e7b42f1efdfdf4879c3f173e59339f9 into b3c38347e4a04cc4841fbc0ecaea2d84(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:15:40,427 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3e7b42f1efdfdf4879c3f173e59339f9: 2024-11-14T09:15:40,427 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., storeName=3e7b42f1efdfdf4879c3f173e59339f9/B, priority=13, startTime=1731575740392; duration=0sec 2024-11-14T09:15:40,427 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:15:40,427 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3e7b42f1efdfdf4879c3f173e59339f9:B 2024-11-14T09:15:40,427 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:15:40,428 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37263 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:15:40,428 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): 3e7b42f1efdfdf4879c3f173e59339f9/C is initiating minor compaction (all files) 2024-11-14T09:15:40,428 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3e7b42f1efdfdf4879c3f173e59339f9/C in TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:40,428 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/0459f6b0fea44368a888bbdbb7d54c94, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/fdb90e36398046a08fe1e483706ee02c, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/8a86429a1f9f41d3b865de240ffeca93] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp, totalSize=36.4 K 2024-11-14T09:15:40,428 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 0459f6b0fea44368a888bbdbb7d54c94, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731575735572 2024-11-14T09:15:40,428 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting fdb90e36398046a08fe1e483706ee02c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731575735808 2024-11-14T09:15:40,429 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a86429a1f9f41d3b865de240ffeca93, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731575738033 2024-11-14T09:15:40,436 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3e7b42f1efdfdf4879c3f173e59339f9#C#compaction#348 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:15:40,437 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/C/c0eb30ccd37446038e83de5dce558dbe is 50, key is test_row_0/C:col10/1731575738727/Put/seqid=0 2024-11-14T09:15:40,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742237_1413 (size=14994) 2024-11-14T09:15:40,439 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:15:40,442 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114aa8a6af015d9436dbcea53e38ddce548_3e7b42f1efdfdf4879c3f173e59339f9 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114aa8a6af015d9436dbcea53e38ddce548_3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:40,445 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/7a64ae7db7ca4a4f9d20047f3a078502, store: [table=TestAcidGuarantees family=A region=3e7b42f1efdfdf4879c3f173e59339f9] 2024-11-14T09:15:40,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742236_1412 (size=4469) 2024-11-14T09:15:40,446 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/7a64ae7db7ca4a4f9d20047f3a078502 is 175, key is test_row_0/A:col10/1731575738809/Put/seqid=0 2024-11-14T09:15:40,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742238_1414 (size=12813) 2024-11-14T09:15:40,451 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/C/c0eb30ccd37446038e83de5dce558dbe as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/c0eb30ccd37446038e83de5dce558dbe 2024-11-14T09:15:40,457 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3e7b42f1efdfdf4879c3f173e59339f9/C of 3e7b42f1efdfdf4879c3f173e59339f9 into c0eb30ccd37446038e83de5dce558dbe(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:15:40,457 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3e7b42f1efdfdf4879c3f173e59339f9: 2024-11-14T09:15:40,458 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., storeName=3e7b42f1efdfdf4879c3f173e59339f9/C, priority=13, startTime=1731575740392; duration=0sec 2024-11-14T09:15:40,458 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:15:40,458 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3e7b42f1efdfdf4879c3f173e59339f9:C 2024-11-14T09:15:40,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742239_1415 (size=39949) 2024-11-14T09:15:40,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:40,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56588 deadline: 1731575800471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:40,485 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:40,485 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:40,485 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:40,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:40,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56502 deadline: 1731575800477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:40,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:40,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:40,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:40,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-14T09:15:40,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:40,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. as already flushing 2024-11-14T09:15:40,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:40,535 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:40,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:40,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:40,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-14T09:15:40,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:40,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56588 deadline: 1731575800594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:40,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:40,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:40,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:40,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:40,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56502 deadline: 1731575800594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:40,603 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:40,603 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:40,603 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:40,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-14T09:15:40,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:40,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. as already flushing 2024-11-14T09:15:40,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:40,688 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:40,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:40,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:40,753 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T09:15:40,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:40,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56588 deadline: 1731575800805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:40,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:40,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:40,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:40,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:40,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56502 deadline: 1731575800814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:40,821 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:40,821 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:40,821 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:40,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-14T09:15:40,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:40,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. as already flushing 2024-11-14T09:15:40,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:40,841 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:40,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:40,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:40,847 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3e7b42f1efdfdf4879c3f173e59339f9#A#compaction#346 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:15:40,847 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/f71e9f7f552a48e88380262f9b00cc39 is 175, key is test_row_0/A:col10/1731575738727/Put/seqid=0 2024-11-14T09:15:40,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742240_1416 (size=31767) 2024-11-14T09:15:40,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-14T09:15:40,859 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=311, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/7a64ae7db7ca4a4f9d20047f3a078502 2024-11-14T09:15:40,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/B/17fcf49d3f1142efb480718326d27407 is 50, key is test_row_0/B:col10/1731575738809/Put/seqid=0 2024-11-14T09:15:40,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742241_1417 (size=12301) 2024-11-14T09:15:40,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-14T09:15:40,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:40,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. as already flushing 2024-11-14T09:15:40,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:40,994 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:40,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:40,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:41,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:41,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56618 deadline: 1731575801005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:41,013 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_0' in 3e7b42f1efdfdf4879c3f173e59339f9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:15:41,013 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:41,013 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:41,013 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:41,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:41,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56588 deadline: 1731575801125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:41,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:41,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:41,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:41,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56502 deadline: 1731575801125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:41,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:41,132 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:41,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:41,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:41,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-14T09:15:41,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:41,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. as already flushing 2024-11-14T09:15:41,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:41,147 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:41,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:41,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:41,255 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/f71e9f7f552a48e88380262f9b00cc39 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/f71e9f7f552a48e88380262f9b00cc39 2024-11-14T09:15:41,259 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3e7b42f1efdfdf4879c3f173e59339f9/A of 3e7b42f1efdfdf4879c3f173e59339f9 into f71e9f7f552a48e88380262f9b00cc39(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:15:41,259 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3e7b42f1efdfdf4879c3f173e59339f9: 2024-11-14T09:15:41,259 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., storeName=3e7b42f1efdfdf4879c3f173e59339f9/A, priority=13, startTime=1731575740392; duration=0sec 2024-11-14T09:15:41,259 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:15:41,259 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3e7b42f1efdfdf4879c3f173e59339f9:A 2024-11-14T09:15:41,272 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/B/17fcf49d3f1142efb480718326d27407 2024-11-14T09:15:41,273 INFO [Thread-1607 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1607. 2024-11-14T09:15:41,273 DEBUG [Thread-1607 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:15:41,273 DEBUG [Thread-1607 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:41,273 DEBUG [Thread-1607 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:41,273 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:15:41,274 INFO [Thread-1603 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1603. 2024-11-14T09:15:41,274 DEBUG [Thread-1603 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:15:41,274 DEBUG [Thread-1603 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:41,274 DEBUG [Thread-1603 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:41,274 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:15:41,277 INFO [Thread-1611 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1611. 2024-11-14T09:15:41,277 DEBUG [Thread-1611 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:15:41,277 DEBUG [Thread-1611 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:41,277 DEBUG [Thread-1611 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:41,277 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:15:41,279 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/C/9bdba6d9588a4a60abec3290f0ba3cf9 is 50, key is test_row_0/C:col10/1731575738809/Put/seqid=0 2024-11-14T09:15:41,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742242_1418 (size=12301) 2024-11-14T09:15:41,283 INFO [Thread-1609 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1609. 2024-11-14T09:15:41,284 DEBUG [Thread-1609 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:15:41,284 DEBUG [Thread-1609 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:41,284 DEBUG [Thread-1609 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:41,284 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:15:41,284 INFO [Thread-1605 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1605. 2024-11-14T09:15:41,284 DEBUG [Thread-1605 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:15:41,284 DEBUG [Thread-1605 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:41,284 DEBUG [Thread-1605 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:41,284 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:15:41,299 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-14T09:15:41,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:41,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. as already flushing 2024-11-14T09:15:41,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:41,300 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:41,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:41,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:41,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-14T09:15:41,452 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-14T09:15:41,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:41,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. as already flushing 2024-11-14T09:15:41,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:41,452 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:41,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:41,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:41,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-14T09:15:41,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:41,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. as already flushing 2024-11-14T09:15:41,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:41,605 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:41,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:41,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:41,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:41,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:41,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56588 deadline: 1731575801645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:41,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56502 deadline: 1731575801645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:41,646 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:41,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:41,646 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:41,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:41,646 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:41,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:15:41,682 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/C/9bdba6d9588a4a60abec3290f0ba3cf9 2024-11-14T09:15:41,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/7a64ae7db7ca4a4f9d20047f3a078502 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/7a64ae7db7ca4a4f9d20047f3a078502 2024-11-14T09:15:41,689 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/7a64ae7db7ca4a4f9d20047f3a078502, entries=200, sequenceid=311, filesize=39.0 K 2024-11-14T09:15:41,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/B/17fcf49d3f1142efb480718326d27407 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/17fcf49d3f1142efb480718326d27407 2024-11-14T09:15:41,692 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/17fcf49d3f1142efb480718326d27407, entries=150, sequenceid=311, filesize=12.0 K 2024-11-14T09:15:41,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/C/9bdba6d9588a4a60abec3290f0ba3cf9 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/9bdba6d9588a4a60abec3290f0ba3cf9 2024-11-14T09:15:41,695 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/9bdba6d9588a4a60abec3290f0ba3cf9, entries=150, sequenceid=311, filesize=12.0 K 2024-11-14T09:15:41,696 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 3e7b42f1efdfdf4879c3f173e59339f9 in 1295ms, sequenceid=311, compaction requested=false 2024-11-14T09:15:41,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3e7b42f1efdfdf4879c3f173e59339f9: 2024-11-14T09:15:41,758 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-14T09:15:41,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:41,758 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 3e7b42f1efdfdf4879c3f173e59339f9 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-14T09:15:41,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e7b42f1efdfdf4879c3f173e59339f9, store=A 2024-11-14T09:15:41,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:15:41,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e7b42f1efdfdf4879c3f173e59339f9, store=B 2024-11-14T09:15:41,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:15:41,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e7b42f1efdfdf4879c3f173e59339f9, store=C 2024-11-14T09:15:41,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:15:41,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e776e8e646d94163baff52813e7e86b9_3e7b42f1efdfdf4879c3f173e59339f9 is 50, key is test_row_0/A:col10/1731575740475/Put/seqid=0 2024-11-14T09:15:41,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742243_1419 (size=12454) 2024-11-14T09:15:42,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:15:42,169 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e776e8e646d94163baff52813e7e86b9_3e7b42f1efdfdf4879c3f173e59339f9 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e776e8e646d94163baff52813e7e86b9_3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:42,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/48d3935f7b6f4467a2d26e690e00494f, store: [table=TestAcidGuarantees family=A region=3e7b42f1efdfdf4879c3f173e59339f9] 2024-11-14T09:15:42,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/48d3935f7b6f4467a2d26e690e00494f is 175, key is test_row_0/A:col10/1731575740475/Put/seqid=0 2024-11-14T09:15:42,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742244_1420 (size=31255) 2024-11-14T09:15:42,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on 3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:42,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. as already flushing 2024-11-14T09:15:42,246 INFO [Thread-1592 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1592. 2024-11-14T09:15:42,246 DEBUG [Thread-1592 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:15:42,246 DEBUG [Thread-1592 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:42,246 DEBUG [Thread-1592 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:42,247 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:15:42,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-14T09:15:42,574 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=332, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/48d3935f7b6f4467a2d26e690e00494f 2024-11-14T09:15:42,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/B/f3c6abd3283348d6a7295162ff857b6f is 50, key is test_row_0/B:col10/1731575740475/Put/seqid=0 2024-11-14T09:15:42,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742245_1421 (size=12301) 2024-11-14T09:15:42,657 INFO [Thread-1596 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1596. 2024-11-14T09:15:42,657 INFO [Thread-1598 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1598. 2024-11-14T09:15:42,657 DEBUG [Thread-1596 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:15:42,657 DEBUG [Thread-1598 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:15:42,657 DEBUG [Thread-1596 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:42,657 DEBUG [Thread-1598 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:42,657 DEBUG [Thread-1596 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:42,657 DEBUG [Thread-1598 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:42,657 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:15:42,657 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:15:42,983 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/B/f3c6abd3283348d6a7295162ff857b6f 2024-11-14T09:15:42,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/C/a03ea2764597410a8b08dd239b8fb97c is 50, key is test_row_0/C:col10/1731575740475/Put/seqid=0 2024-11-14T09:15:42,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742246_1422 (size=12301) 2024-11-14T09:15:43,026 INFO [Thread-1594 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1594. 2024-11-14T09:15:43,026 DEBUG [Thread-1594 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:15:43,026 DEBUG [Thread-1594 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:43,026 DEBUG [Thread-1594 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:43,026 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:15:43,393 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/C/a03ea2764597410a8b08dd239b8fb97c 2024-11-14T09:15:43,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/48d3935f7b6f4467a2d26e690e00494f as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/48d3935f7b6f4467a2d26e690e00494f 2024-11-14T09:15:43,399 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/48d3935f7b6f4467a2d26e690e00494f, entries=150, sequenceid=332, filesize=30.5 K 2024-11-14T09:15:43,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/B/f3c6abd3283348d6a7295162ff857b6f as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/f3c6abd3283348d6a7295162ff857b6f 2024-11-14T09:15:43,402 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/f3c6abd3283348d6a7295162ff857b6f, entries=150, sequenceid=332, filesize=12.0 K 2024-11-14T09:15:43,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/C/a03ea2764597410a8b08dd239b8fb97c as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/a03ea2764597410a8b08dd239b8fb97c 2024-11-14T09:15:43,405 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/a03ea2764597410a8b08dd239b8fb97c, entries=150, sequenceid=332, filesize=12.0 K 2024-11-14T09:15:43,406 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=26.84 KB/27480 for 3e7b42f1efdfdf4879c3f173e59339f9 in 1648ms, sequenceid=332, compaction requested=true 2024-11-14T09:15:43,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 3e7b42f1efdfdf4879c3f173e59339f9: 2024-11-14T09:15:43,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:43,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-14T09:15:43,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-14T09:15:43,408 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-14T09:15:43,408 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1790 sec 2024-11-14T09:15:43,410 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 3.1830 sec 2024-11-14T09:15:44,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-14T09:15:44,396 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:15:44,561 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83f56b55f2af:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/2e05102ca2e1427dabaffc12762cce69, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/380ef9e69234416cbea39c92d08284fd, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/d222198bbbe943a78fc9c48d98443291, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/6ae67e5ade1246fbac5081fa0249fd1d, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/385d8245cf934bcf87169ff16dac88d6, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/2cb74e9db25d4c77adb4e29a612c755c, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/d649935043724b438195a8817ff9868c, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/fc638de742f64c3e9eef03fe047e63df, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/13ffeb1b07834f8797a073fa82ae7e30, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/e6b13cdd5f054d63a0dc4f4a9f2f5073, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/d30bb31b4679475a97ca5bf3958cf6cf, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/1195d60340f64a19b2502a6ed04a5a8e, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/93c3a59eb8d64eea9b3653b6ad5136e0, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/3ad25bbfbd634429a9188833c9bde061, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/917236fcda7e4e7e84941e448ac8f55d] to archive 2024-11-14T09:15:44,563 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83f56b55f2af:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T09:15:44,563 WARN [regionserver/83f56b55f2af:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-11-14T09:15:44,564 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83f56b55f2af:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/2e05102ca2e1427dabaffc12762cce69 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/2e05102ca2e1427dabaffc12762cce69 2024-11-14T09:15:44,565 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83f56b55f2af:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/380ef9e69234416cbea39c92d08284fd to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/380ef9e69234416cbea39c92d08284fd 2024-11-14T09:15:44,566 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83f56b55f2af:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/d222198bbbe943a78fc9c48d98443291 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/d222198bbbe943a78fc9c48d98443291 2024-11-14T09:15:44,566 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83f56b55f2af:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/6ae67e5ade1246fbac5081fa0249fd1d to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/6ae67e5ade1246fbac5081fa0249fd1d 2024-11-14T09:15:44,567 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83f56b55f2af:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/385d8245cf934bcf87169ff16dac88d6 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/385d8245cf934bcf87169ff16dac88d6 2024-11-14T09:15:44,568 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83f56b55f2af:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/2cb74e9db25d4c77adb4e29a612c755c to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/2cb74e9db25d4c77adb4e29a612c755c 2024-11-14T09:15:44,569 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83f56b55f2af:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/d649935043724b438195a8817ff9868c to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/d649935043724b438195a8817ff9868c 2024-11-14T09:15:44,570 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83f56b55f2af:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/fc638de742f64c3e9eef03fe047e63df to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/fc638de742f64c3e9eef03fe047e63df 2024-11-14T09:15:44,570 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83f56b55f2af:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/13ffeb1b07834f8797a073fa82ae7e30 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/13ffeb1b07834f8797a073fa82ae7e30 2024-11-14T09:15:44,571 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83f56b55f2af:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/e6b13cdd5f054d63a0dc4f4a9f2f5073 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/e6b13cdd5f054d63a0dc4f4a9f2f5073 2024-11-14T09:15:44,572 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83f56b55f2af:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/d30bb31b4679475a97ca5bf3958cf6cf to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/d30bb31b4679475a97ca5bf3958cf6cf 2024-11-14T09:15:44,573 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83f56b55f2af:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/1195d60340f64a19b2502a6ed04a5a8e to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/1195d60340f64a19b2502a6ed04a5a8e 2024-11-14T09:15:44,573 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83f56b55f2af:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/93c3a59eb8d64eea9b3653b6ad5136e0 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/93c3a59eb8d64eea9b3653b6ad5136e0 2024-11-14T09:15:44,574 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83f56b55f2af:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/3ad25bbfbd634429a9188833c9bde061 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/3ad25bbfbd634429a9188833c9bde061 2024-11-14T09:15:44,575 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83f56b55f2af:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/917236fcda7e4e7e84941e448ac8f55d to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/917236fcda7e4e7e84941e448ac8f55d 2024-11-14T09:15:44,577 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83f56b55f2af:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/ff9f097805ea4ab6bce8343d3c31b98c, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/daddea011d054a3394e9581dded8dd1b, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/df5979a7878c440c891db862a393d0f2, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/e4779a7fb55f4615bb7a903b88502c26, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/5e5185c4ecc846d69e929395891d25da, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/0e48131d01d143eca9996e938a879208, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/66a303432a87471aa7cc19548d1e3bdc, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/3314466ada124ff0a1d3c7b7b8f70870, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/9e207e38bd90454387ef9941896740a8, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/e8d81ce9fe0442028d6abeb26ad1c554, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/641b258c7ad244f29a0c22ce3d53bd36, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/057521bf67874bcda8f04ccc324e832d, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/8a8237991ccd4fe7a44e8b66ad7b3d23, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/3bc1ff2c26a745dbbce659947d90cff5, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/3601016207ee4ad9b21822e1e64af55d] to archive 2024-11-14T09:15:44,577 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83f56b55f2af:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T09:15:44,578 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83f56b55f2af:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/ff9f097805ea4ab6bce8343d3c31b98c to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/ff9f097805ea4ab6bce8343d3c31b98c 2024-11-14T09:15:44,579 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83f56b55f2af:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/daddea011d054a3394e9581dded8dd1b to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/daddea011d054a3394e9581dded8dd1b 2024-11-14T09:15:44,580 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83f56b55f2af:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/df5979a7878c440c891db862a393d0f2 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/df5979a7878c440c891db862a393d0f2 2024-11-14T09:15:44,581 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83f56b55f2af:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/e4779a7fb55f4615bb7a903b88502c26 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/e4779a7fb55f4615bb7a903b88502c26 2024-11-14T09:15:44,581 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83f56b55f2af:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/5e5185c4ecc846d69e929395891d25da to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/5e5185c4ecc846d69e929395891d25da 2024-11-14T09:15:44,582 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83f56b55f2af:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/0e48131d01d143eca9996e938a879208 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/0e48131d01d143eca9996e938a879208 2024-11-14T09:15:44,583 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83f56b55f2af:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/66a303432a87471aa7cc19548d1e3bdc to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/66a303432a87471aa7cc19548d1e3bdc 2024-11-14T09:15:44,584 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83f56b55f2af:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/3314466ada124ff0a1d3c7b7b8f70870 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/3314466ada124ff0a1d3c7b7b8f70870 2024-11-14T09:15:44,584 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83f56b55f2af:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/9e207e38bd90454387ef9941896740a8 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/9e207e38bd90454387ef9941896740a8 2024-11-14T09:15:44,585 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83f56b55f2af:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/e8d81ce9fe0442028d6abeb26ad1c554 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/e8d81ce9fe0442028d6abeb26ad1c554 2024-11-14T09:15:44,586 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83f56b55f2af:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/641b258c7ad244f29a0c22ce3d53bd36 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/641b258c7ad244f29a0c22ce3d53bd36 2024-11-14T09:15:44,587 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83f56b55f2af:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/057521bf67874bcda8f04ccc324e832d to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/057521bf67874bcda8f04ccc324e832d 2024-11-14T09:15:44,587 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83f56b55f2af:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/8a8237991ccd4fe7a44e8b66ad7b3d23 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/8a8237991ccd4fe7a44e8b66ad7b3d23 2024-11-14T09:15:44,588 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83f56b55f2af:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/3bc1ff2c26a745dbbce659947d90cff5 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/3bc1ff2c26a745dbbce659947d90cff5 2024-11-14T09:15:44,589 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83f56b55f2af:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/3601016207ee4ad9b21822e1e64af55d to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/3601016207ee4ad9b21822e1e64af55d 2024-11-14T09:15:44,591 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83f56b55f2af:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/45e8892f87d4461a82b1bcc88b7c5cc2, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/660d56825a6e42a6b50b79fd05542481, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/8c82de3224454519b6c104ce8a63ae70, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/58f7cbb3f54b424cb271771435f75a5e, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/14907c6607f7409da44ed7257aa43bab, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/d76752d352984488bea5d6725bc1619f, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/c4a4e87d25f44ada8cc09f7b6c3912e5, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/6a3e9821aae74556a4ee90fa639965f2, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/8da9179cc0fc4f80ae03d01d58345674, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/91e325f61998490bba77fa647b708739, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/54867843ea2547f79b47ca63461c7529, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/0459f6b0fea44368a888bbdbb7d54c94, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/4c67b3a114f546e79ab39e0ad87576d1, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/fdb90e36398046a08fe1e483706ee02c, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/8a86429a1f9f41d3b865de240ffeca93] to archive 2024-11-14T09:15:44,591 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83f56b55f2af:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T09:15:44,592 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83f56b55f2af:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/45e8892f87d4461a82b1bcc88b7c5cc2 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/45e8892f87d4461a82b1bcc88b7c5cc2 2024-11-14T09:15:44,593 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83f56b55f2af:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/660d56825a6e42a6b50b79fd05542481 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/660d56825a6e42a6b50b79fd05542481 2024-11-14T09:15:44,594 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83f56b55f2af:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/8c82de3224454519b6c104ce8a63ae70 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/8c82de3224454519b6c104ce8a63ae70 2024-11-14T09:15:44,594 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83f56b55f2af:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/58f7cbb3f54b424cb271771435f75a5e to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/58f7cbb3f54b424cb271771435f75a5e 2024-11-14T09:15:44,595 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83f56b55f2af:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/14907c6607f7409da44ed7257aa43bab to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/14907c6607f7409da44ed7257aa43bab 2024-11-14T09:15:44,596 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83f56b55f2af:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/d76752d352984488bea5d6725bc1619f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/d76752d352984488bea5d6725bc1619f 2024-11-14T09:15:44,597 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83f56b55f2af:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/c4a4e87d25f44ada8cc09f7b6c3912e5 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/c4a4e87d25f44ada8cc09f7b6c3912e5 2024-11-14T09:15:44,597 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83f56b55f2af:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/6a3e9821aae74556a4ee90fa639965f2 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/6a3e9821aae74556a4ee90fa639965f2 2024-11-14T09:15:44,598 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83f56b55f2af:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/8da9179cc0fc4f80ae03d01d58345674 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/8da9179cc0fc4f80ae03d01d58345674 2024-11-14T09:15:44,599 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83f56b55f2af:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/91e325f61998490bba77fa647b708739 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/91e325f61998490bba77fa647b708739 2024-11-14T09:15:44,599 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83f56b55f2af:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/54867843ea2547f79b47ca63461c7529 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/54867843ea2547f79b47ca63461c7529 2024-11-14T09:15:44,600 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83f56b55f2af:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/0459f6b0fea44368a888bbdbb7d54c94 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/0459f6b0fea44368a888bbdbb7d54c94 2024-11-14T09:15:44,601 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83f56b55f2af:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/4c67b3a114f546e79ab39e0ad87576d1 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/4c67b3a114f546e79ab39e0ad87576d1 2024-11-14T09:15:44,601 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83f56b55f2af:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/fdb90e36398046a08fe1e483706ee02c to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/fdb90e36398046a08fe1e483706ee02c 2024-11-14T09:15:44,602 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83f56b55f2af:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/8a86429a1f9f41d3b865de240ffeca93 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/8a86429a1f9f41d3b865de240ffeca93 2024-11-14T09:15:50,467 INFO [Thread-1600 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1600. 2024-11-14T09:15:50,467 DEBUG [Thread-1600 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:15:50,467 DEBUG [Thread-1600 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:50,467 DEBUG [Thread-1600 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:50,467 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:15:50,467 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T09:15:50,467 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 112 2024-11-14T09:15:50,467 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 118 2024-11-14T09:15:50,467 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 20 2024-11-14T09:15:50,467 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-11-14T09:15:50,467 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 12 2024-11-14T09:15:50,467 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T09:15:50,467 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T09:15:50,467 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1001 2024-11-14T09:15:50,467 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3003 rows 2024-11-14T09:15:50,467 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1012 2024-11-14T09:15:50,467 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3031 rows 2024-11-14T09:15:50,467 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1007 2024-11-14T09:15:50,467 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3021 rows 2024-11-14T09:15:50,467 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1012 2024-11-14T09:15:50,467 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3023 rows 2024-11-14T09:15:50,467 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1016 2024-11-14T09:15:50,467 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3029 rows 2024-11-14T09:15:50,468 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T09:15:50,468 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T09:15:50,468 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:50,468 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:50,468 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:15:50,468 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-14T09:15:50,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T09:15:50,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-14T09:15:50,471 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731575750471"}]},"ts":"1731575750471"} 2024-11-14T09:15:50,472 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T09:15:50,472 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T09:15:50,473 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T09:15:50,474 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3e7b42f1efdfdf4879c3f173e59339f9, UNASSIGN}] 2024-11-14T09:15:50,474 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3e7b42f1efdfdf4879c3f173e59339f9, UNASSIGN 2024-11-14T09:15:50,475 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=3e7b42f1efdfdf4879c3f173e59339f9, regionState=CLOSING, regionLocation=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:50,476 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3e7b42f1efdfdf4879c3f173e59339f9, UNASSIGN because future has completed 2024-11-14T09:15:50,476 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T09:15:50,476 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393}] 2024-11-14T09:15:50,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-14T09:15:50,628 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(122): Close 3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:50,628 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T09:15:50,628 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1722): Closing 3e7b42f1efdfdf4879c3f173e59339f9, disabling compactions & flushes 2024-11-14T09:15:50,628 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:50,628 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:50,628 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. after waiting 0 ms 2024-11-14T09:15:50,628 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:50,628 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(2902): Flushing 3e7b42f1efdfdf4879c3f173e59339f9 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-14T09:15:50,629 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e7b42f1efdfdf4879c3f173e59339f9, store=A 2024-11-14T09:15:50,629 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:15:50,629 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e7b42f1efdfdf4879c3f173e59339f9, store=B 2024-11-14T09:15:50,629 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:15:50,629 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e7b42f1efdfdf4879c3f173e59339f9, store=C 2024-11-14T09:15:50,629 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:15:50,634 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114ef0718273e05408794c90ad4a6d2b3e6_3e7b42f1efdfdf4879c3f173e59339f9 is 50, key is test_row_0/A:col10/1731575750465/Put/seqid=0 2024-11-14T09:15:50,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742247_1423 (size=12454) 2024-11-14T09:15:50,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-14T09:15:51,037 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:15:51,041 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114ef0718273e05408794c90ad4a6d2b3e6_3e7b42f1efdfdf4879c3f173e59339f9 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ef0718273e05408794c90ad4a6d2b3e6_3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:51,041 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/ba98b44e46ae466ea7c778359aad8669, store: [table=TestAcidGuarantees family=A region=3e7b42f1efdfdf4879c3f173e59339f9] 2024-11-14T09:15:51,042 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/ba98b44e46ae466ea7c778359aad8669 is 175, key is test_row_0/A:col10/1731575750465/Put/seqid=0 2024-11-14T09:15:51,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742248_1424 (size=31255) 2024-11-14T09:15:51,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-14T09:15:51,446 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=340, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/ba98b44e46ae466ea7c778359aad8669 2024-11-14T09:15:51,451 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/B/de2e35a1b1be4f8f89e642e7e820c736 is 50, key is test_row_0/B:col10/1731575750465/Put/seqid=0 2024-11-14T09:15:51,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742249_1425 (size=12301) 2024-11-14T09:15:51,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-14T09:15:51,855 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/B/de2e35a1b1be4f8f89e642e7e820c736 2024-11-14T09:15:51,861 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/C/1794f3cc005c419eb9d838e0e7bdd506 is 50, key is test_row_0/C:col10/1731575750465/Put/seqid=0 2024-11-14T09:15:51,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742250_1426 (size=12301) 2024-11-14T09:15:52,264 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/C/1794f3cc005c419eb9d838e0e7bdd506 2024-11-14T09:15:52,268 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/A/ba98b44e46ae466ea7c778359aad8669 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/ba98b44e46ae466ea7c778359aad8669 2024-11-14T09:15:52,271 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/ba98b44e46ae466ea7c778359aad8669, entries=150, sequenceid=340, filesize=30.5 K 2024-11-14T09:15:52,272 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/B/de2e35a1b1be4f8f89e642e7e820c736 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/de2e35a1b1be4f8f89e642e7e820c736 2024-11-14T09:15:52,275 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/de2e35a1b1be4f8f89e642e7e820c736, entries=150, sequenceid=340, filesize=12.0 K 2024-11-14T09:15:52,275 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/.tmp/C/1794f3cc005c419eb9d838e0e7bdd506 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/1794f3cc005c419eb9d838e0e7bdd506 2024-11-14T09:15:52,277 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/1794f3cc005c419eb9d838e0e7bdd506, entries=150, sequenceid=340, filesize=12.0 K 2024-11-14T09:15:52,278 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 3e7b42f1efdfdf4879c3f173e59339f9 in 1650ms, sequenceid=340, compaction requested=true 2024-11-14T09:15:52,282 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/recovered.edits/343.seqid, newMaxSeqId=343, maxSeqId=4 2024-11-14T09:15:52,282 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. 2024-11-14T09:15:52,282 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1676): Region close journal for 3e7b42f1efdfdf4879c3f173e59339f9: Waiting for close lock at 1731575750628Running coprocessor pre-close hooks at 1731575750628Disabling compacts and flushes for region at 1731575750628Disabling writes for close at 1731575750628Obtaining lock to block concurrent updates at 1731575750628Preparing flush snapshotting stores in 3e7b42f1efdfdf4879c3f173e59339f9 at 1731575750628Finished memstore snapshotting TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731575750629 (+1 ms)Flushing stores of TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9. at 1731575750629Flushing 3e7b42f1efdfdf4879c3f173e59339f9/A: creating writer at 1731575750629Flushing mob file 3e7b42f1efdfdf4879c3f173e59339f9/A: appending metadata at 1731575750634 (+5 ms)Flushing mob file 3e7b42f1efdfdf4879c3f173e59339f9/A: closing flushed file at 1731575750634Flushing 3e7b42f1efdfdf4879c3f173e59339f9/A: appending metadata at 1731575751042 (+408 ms)Flushing 3e7b42f1efdfdf4879c3f173e59339f9/A: closing flushed file at 1731575751042Flushing 3e7b42f1efdfdf4879c3f173e59339f9/B: creating writer at 1731575751449 (+407 ms)Flushing 3e7b42f1efdfdf4879c3f173e59339f9/B: appending metadata at 1731575751451 (+2 ms)Flushing 3e7b42f1efdfdf4879c3f173e59339f9/B: closing flushed file at 1731575751451Flushing 3e7b42f1efdfdf4879c3f173e59339f9/C: creating writer at 1731575751858 (+407 ms)Flushing 3e7b42f1efdfdf4879c3f173e59339f9/C: appending metadata at 1731575751860 (+2 ms)Flushing 3e7b42f1efdfdf4879c3f173e59339f9/C: closing flushed file at 1731575751860Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@184b3f13: reopening flushed file at 1731575752268 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ba1f480: reopening flushed file at 1731575752271 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3cfdd60e: reopening flushed file at 1731575752275 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 3e7b42f1efdfdf4879c3f173e59339f9 in 1650ms, sequenceid=340, compaction requested=true at 1731575752278 (+3 ms)Writing region close event to WAL at 1731575752279 (+1 ms)Running coprocessor post-close hooks at 1731575752282 (+3 ms)Closed at 1731575752282 2024-11-14T09:15:52,284 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(157): Closed 3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:52,284 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=3e7b42f1efdfdf4879c3f173e59339f9, regionState=CLOSED 2024-11-14T09:15:52,286 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=112, ppid=111, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 because future has completed 2024-11-14T09:15:52,287 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-14T09:15:52,288 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; CloseRegionProcedure 3e7b42f1efdfdf4879c3f173e59339f9, server=83f56b55f2af,39927,1731575622393 in 1.8100 sec 2024-11-14T09:15:52,289 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=111, resume processing ppid=110 2024-11-14T09:15:52,289 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, ppid=110, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3e7b42f1efdfdf4879c3f173e59339f9, UNASSIGN in 1.8140 sec 2024-11-14T09:15:52,290 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-14T09:15:52,290 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8160 sec 2024-11-14T09:15:52,292 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731575752291"}]},"ts":"1731575752291"} 2024-11-14T09:15:52,293 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T09:15:52,293 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T09:15:52,294 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8240 sec 2024-11-14T09:15:52,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-14T09:15:52,616 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:15:52,616 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-14T09:15:52,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:15:52,618 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=113, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:15:52,618 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=113, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:15:52,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-14T09:15:52,621 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:52,623 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A, FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B, FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C, FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/recovered.edits] 2024-11-14T09:15:52,625 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/48d3935f7b6f4467a2d26e690e00494f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/48d3935f7b6f4467a2d26e690e00494f 2024-11-14T09:15:52,626 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/7a64ae7db7ca4a4f9d20047f3a078502 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/7a64ae7db7ca4a4f9d20047f3a078502 2024-11-14T09:15:52,627 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/ba98b44e46ae466ea7c778359aad8669 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/ba98b44e46ae466ea7c778359aad8669 2024-11-14T09:15:52,628 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/f71e9f7f552a48e88380262f9b00cc39 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/A/f71e9f7f552a48e88380262f9b00cc39 2024-11-14T09:15:52,630 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/17fcf49d3f1142efb480718326d27407 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/17fcf49d3f1142efb480718326d27407 2024-11-14T09:15:52,631 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/b3c38347e4a04cc4841fbc0ecaea2d84 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/b3c38347e4a04cc4841fbc0ecaea2d84 2024-11-14T09:15:52,632 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/de2e35a1b1be4f8f89e642e7e820c736 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/de2e35a1b1be4f8f89e642e7e820c736 2024-11-14T09:15:52,633 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/f3c6abd3283348d6a7295162ff857b6f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/B/f3c6abd3283348d6a7295162ff857b6f 2024-11-14T09:15:52,634 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/1794f3cc005c419eb9d838e0e7bdd506 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/1794f3cc005c419eb9d838e0e7bdd506 2024-11-14T09:15:52,635 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/9bdba6d9588a4a60abec3290f0ba3cf9 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/9bdba6d9588a4a60abec3290f0ba3cf9 2024-11-14T09:15:52,636 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/a03ea2764597410a8b08dd239b8fb97c to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/a03ea2764597410a8b08dd239b8fb97c 2024-11-14T09:15:52,637 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/c0eb30ccd37446038e83de5dce558dbe to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/C/c0eb30ccd37446038e83de5dce558dbe 2024-11-14T09:15:52,639 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/recovered.edits/343.seqid to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9/recovered.edits/343.seqid 2024-11-14T09:15:52,640 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:52,640 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T09:15:52,640 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T09:15:52,641 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-14T09:15:52,643 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411140121e7a0074c4ea2b59a956a41d6702a_3e7b42f1efdfdf4879c3f173e59339f9 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411140121e7a0074c4ea2b59a956a41d6702a_3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:52,644 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111426d16803292e4c4091f7b5c2a8905881_3e7b42f1efdfdf4879c3f173e59339f9 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111426d16803292e4c4091f7b5c2a8905881_3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:52,646 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143c865bd73a424cefbb8e11a9ecab003d_3e7b42f1efdfdf4879c3f173e59339f9 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143c865bd73a424cefbb8e11a9ecab003d_3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:52,647 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111441cdc958f94549788203656293259a0b_3e7b42f1efdfdf4879c3f173e59339f9 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111441cdc958f94549788203656293259a0b_3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:52,647 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145313ba8666cc4e079ec2de37041caf0b_3e7b42f1efdfdf4879c3f173e59339f9 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145313ba8666cc4e079ec2de37041caf0b_3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:52,648 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145527d028902a4516a0a53503ce38f0fb_3e7b42f1efdfdf4879c3f173e59339f9 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145527d028902a4516a0a53503ce38f0fb_3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:52,649 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114600b3eec90cd4763aed0795b0525fe09_3e7b42f1efdfdf4879c3f173e59339f9 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114600b3eec90cd4763aed0795b0525fe09_3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:52,650 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111489ff447c450e44d888d2df5bd3610388_3e7b42f1efdfdf4879c3f173e59339f9 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111489ff447c450e44d888d2df5bd3610388_3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:52,651 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411148bbeb9c2c4f74e0d936ef7161147dbae_3e7b42f1efdfdf4879c3f173e59339f9 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411148bbeb9c2c4f74e0d936ef7161147dbae_3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:52,652 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111497008c39b4cb44ad8640cbaa6a75e4a8_3e7b42f1efdfdf4879c3f173e59339f9 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111497008c39b4cb44ad8640cbaa6a75e4a8_3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:52,653 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114aa8a6af015d9436dbcea53e38ddce548_3e7b42f1efdfdf4879c3f173e59339f9 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114aa8a6af015d9436dbcea53e38ddce548_3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:52,653 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e776e8e646d94163baff52813e7e86b9_3e7b42f1efdfdf4879c3f173e59339f9 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e776e8e646d94163baff52813e7e86b9_3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:52,654 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e8fc6d2adab2478fa4d02c54c7a8e462_3e7b42f1efdfdf4879c3f173e59339f9 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e8fc6d2adab2478fa4d02c54c7a8e462_3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:52,655 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ef0718273e05408794c90ad4a6d2b3e6_3e7b42f1efdfdf4879c3f173e59339f9 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ef0718273e05408794c90ad4a6d2b3e6_3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:52,656 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114fbded4556f0949f0af602f76cc3935b5_3e7b42f1efdfdf4879c3f173e59339f9 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114fbded4556f0949f0af602f76cc3935b5_3e7b42f1efdfdf4879c3f173e59339f9 2024-11-14T09:15:52,656 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T09:15:52,658 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=113, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:15:52,660 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T09:15:52,661 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T09:15:52,662 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=113, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:15:52,662 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T09:15:52,662 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731575752662"}]},"ts":"9223372036854775807"} 2024-11-14T09:15:52,664 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T09:15:52,664 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 3e7b42f1efdfdf4879c3f173e59339f9, NAME => 'TestAcidGuarantees,,1731575718080.3e7b42f1efdfdf4879c3f173e59339f9.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T09:15:52,664 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T09:15:52,664 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731575752664"}]},"ts":"9223372036854775807"} 2024-11-14T09:15:52,665 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T09:15:52,666 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=113, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:15:52,666 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 50 msec 2024-11-14T09:15:52,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-14T09:15:52,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T09:15:52,725 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:15:52,734 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=249 (was 246) - Thread LEAK? -, OpenFileDescriptor=443 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=455 (was 450) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6235 (was 6787) 2024-11-14T09:15:52,742 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=249, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=455, ProcessCount=11, AvailableMemoryMB=6234 2024-11-14T09:15:52,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T09:15:52,744 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T09:15:52,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=114, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T09:15:52,746 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T09:15:52,746 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:15:52,746 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 114 2024-11-14T09:15:52,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-14T09:15:52,746 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T09:15:52,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742251_1427 (size=1061) 2024-11-14T09:15:52,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-14T09:15:53,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-14T09:15:53,153 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db 2024-11-14T09:15:53,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742252_1428 (size=53) 2024-11-14T09:15:53,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-14T09:15:53,558 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:15:53,559 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing add92018c22d01dad02f6ba9a2020355, disabling compactions & flushes 2024-11-14T09:15:53,559 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:15:53,559 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:15:53,559 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. after waiting 0 ms 2024-11-14T09:15:53,559 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:15:53,559 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:15:53,559 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for add92018c22d01dad02f6ba9a2020355: Waiting for close lock at 1731575753559Disabling compacts and flushes for region at 1731575753559Disabling writes for close at 1731575753559Writing region close event to WAL at 1731575753559Closed at 1731575753559 2024-11-14T09:15:53,560 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T09:15:53,560 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731575753560"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731575753560"}]},"ts":"1731575753560"} 2024-11-14T09:15:53,561 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T09:15:53,562 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T09:15:53,562 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731575753562"}]},"ts":"1731575753562"} 2024-11-14T09:15:53,563 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T09:15:53,563 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=add92018c22d01dad02f6ba9a2020355, ASSIGN}] 2024-11-14T09:15:53,564 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=add92018c22d01dad02f6ba9a2020355, ASSIGN 2024-11-14T09:15:53,565 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=add92018c22d01dad02f6ba9a2020355, ASSIGN; state=OFFLINE, location=83f56b55f2af,39927,1731575622393; forceNewPlan=false, retain=false 2024-11-14T09:15:53,715 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=add92018c22d01dad02f6ba9a2020355, regionState=OPENING, regionLocation=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:53,717 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=add92018c22d01dad02f6ba9a2020355, ASSIGN because future has completed 2024-11-14T09:15:53,717 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; OpenRegionProcedure add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393}] 2024-11-14T09:15:53,871 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:15:53,871 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7752): Opening region: {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} 2024-11-14T09:15:53,872 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees add92018c22d01dad02f6ba9a2020355 2024-11-14T09:15:53,872 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:15:53,873 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7794): checking encryption for add92018c22d01dad02f6ba9a2020355 2024-11-14T09:15:53,873 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7797): checking classloading for add92018c22d01dad02f6ba9a2020355 2024-11-14T09:15:53,873 INFO [StoreOpener-add92018c22d01dad02f6ba9a2020355-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region add92018c22d01dad02f6ba9a2020355 2024-11-14T09:15:53,874 INFO [StoreOpener-add92018c22d01dad02f6ba9a2020355-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T09:15:53,875 INFO [StoreOpener-add92018c22d01dad02f6ba9a2020355-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region add92018c22d01dad02f6ba9a2020355 columnFamilyName A 2024-11-14T09:15:53,875 DEBUG [StoreOpener-add92018c22d01dad02f6ba9a2020355-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:15:53,875 INFO [StoreOpener-add92018c22d01dad02f6ba9a2020355-1 {}] regionserver.HStore(327): Store=add92018c22d01dad02f6ba9a2020355/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:15:53,875 INFO [StoreOpener-add92018c22d01dad02f6ba9a2020355-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region add92018c22d01dad02f6ba9a2020355 2024-11-14T09:15:53,876 INFO [StoreOpener-add92018c22d01dad02f6ba9a2020355-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T09:15:53,876 INFO [StoreOpener-add92018c22d01dad02f6ba9a2020355-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region add92018c22d01dad02f6ba9a2020355 columnFamilyName B 2024-11-14T09:15:53,876 DEBUG [StoreOpener-add92018c22d01dad02f6ba9a2020355-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:15:53,876 INFO [StoreOpener-add92018c22d01dad02f6ba9a2020355-1 {}] regionserver.HStore(327): Store=add92018c22d01dad02f6ba9a2020355/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:15:53,876 INFO [StoreOpener-add92018c22d01dad02f6ba9a2020355-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region add92018c22d01dad02f6ba9a2020355 2024-11-14T09:15:53,877 INFO [StoreOpener-add92018c22d01dad02f6ba9a2020355-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T09:15:53,877 INFO [StoreOpener-add92018c22d01dad02f6ba9a2020355-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region add92018c22d01dad02f6ba9a2020355 columnFamilyName C 2024-11-14T09:15:53,877 DEBUG [StoreOpener-add92018c22d01dad02f6ba9a2020355-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:15:53,878 INFO [StoreOpener-add92018c22d01dad02f6ba9a2020355-1 {}] regionserver.HStore(327): Store=add92018c22d01dad02f6ba9a2020355/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:15:53,878 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:15:53,878 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1038): replaying wal for add92018c22d01dad02f6ba9a2020355 2024-11-14T09:15:53,878 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355 2024-11-14T09:15:53,878 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355 2024-11-14T09:15:53,879 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1048): stopping wal replay for add92018c22d01dad02f6ba9a2020355 2024-11-14T09:15:53,879 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1060): Cleaning up temporary data for add92018c22d01dad02f6ba9a2020355 2024-11-14T09:15:53,879 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T09:15:53,880 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1093): writing seq id for add92018c22d01dad02f6ba9a2020355 2024-11-14T09:15:53,881 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T09:15:53,882 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1114): Opened add92018c22d01dad02f6ba9a2020355; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66692879, jitterRate=-0.006198659539222717}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T09:15:53,882 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1122): Running coprocessor post-open hooks for add92018c22d01dad02f6ba9a2020355 2024-11-14T09:15:53,882 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1006): Region open journal for add92018c22d01dad02f6ba9a2020355: Running coprocessor pre-open hook at 1731575753873Writing region info on filesystem at 1731575753873Initializing all the Stores at 1731575753873Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731575753873Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731575753873Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731575753873Cleaning up temporary data from old regions at 1731575753879 (+6 ms)Running coprocessor post-open hooks at 1731575753882 (+3 ms)Region opened successfully at 1731575753882 2024-11-14T09:15:53,883 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., pid=116, masterSystemTime=1731575753869 2024-11-14T09:15:53,884 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:15:53,884 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:15:53,885 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=add92018c22d01dad02f6ba9a2020355, regionState=OPEN, openSeqNum=2, regionLocation=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:53,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-14T09:15:53,886 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=116, ppid=115, state=RUNNABLE, hasLock=false; OpenRegionProcedure add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 because future has completed 2024-11-14T09:15:53,888 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-14T09:15:53,888 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; OpenRegionProcedure add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 in 170 msec 2024-11-14T09:15:53,889 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=115, resume processing ppid=114 2024-11-14T09:15:53,889 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, ppid=114, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=add92018c22d01dad02f6ba9a2020355, ASSIGN in 325 msec 2024-11-14T09:15:53,890 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T09:15:53,890 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731575753890"}]},"ts":"1731575753890"} 2024-11-14T09:15:53,891 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T09:15:53,892 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T09:15:53,893 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1480 sec 2024-11-14T09:15:54,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-14T09:15:54,895 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:15:54,896 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@be1708d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:54,896 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:15:54,896 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:15:54,897 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:15:54,897 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:15:54,897 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:15:54,897 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77fbc82e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:54,897 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:15:54,898 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:15:54,898 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:54,899 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47748, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:15:54,899 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7adbd3ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:54,899 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:15:54,900 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:15:54,900 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:15:54,901 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59486, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:15:54,901 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T09:15:54,902 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 83f56b55f2af,45325,1731575621660 2024-11-14T09:15:54,902 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1da8c2d4 2024-11-14T09:15:54,902 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T09:15:54,903 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47762, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T09:15:54,904 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16fe0696, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:54,904 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:15:54,904 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:15:54,904 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:15:54,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:15:54,905 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:15:54,905 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2843f3c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:54,905 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:15:54,905 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:15:54,905 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:54,905 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47784, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:15:54,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@684178e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:54,906 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59de117f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:54,906 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:15:54,906 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:15:54,907 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:15:54,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:15:54,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:15:54,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7853eac4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:54,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:15:54,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:15:54,908 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:54,908 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47802, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:15:54,908 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@788ccfe4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:54,909 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@472a8168, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:54,909 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:15:54,909 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:15:54,909 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:15:54,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:15:54,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:15:54,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7af0ae84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:54,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:15:54,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:15:54,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:54,910 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47828, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:15:54,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20c3ad79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:54,911 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f010827, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:54,911 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:15:54,911 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:15:54,912 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:15:54,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:15:54,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:15:54,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e6ed2f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:54,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:15:54,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:15:54,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:54,913 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47854, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:15:54,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f2f3029, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:54,914 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2822c33e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:54,914 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:15:54,914 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:15:54,914 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:15:54,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:15:54,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:15:54,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72eb882e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:54,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:15:54,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:15:54,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:54,915 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47880, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:15:54,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39798ca7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:54,916 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bc5e64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:54,916 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:15:54,916 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:15:54,916 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:15:54,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:15:54,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:15:54,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c333d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:54,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:15:54,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:15:54,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:54,917 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47906, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:15:54,918 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64f32c9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:54,918 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42daa6ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:54,918 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:15:54,918 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:15:54,919 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:15:54,919 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:15:54,919 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:15:54,919 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4277c86a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:54,919 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:15:54,919 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:15:54,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:54,920 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47924, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:15:54,921 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43a845d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:54,921 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@366fb241, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:54,921 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:15:54,921 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:15:54,922 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:15:54,922 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:15:54,922 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:15:54,922 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40fd6eda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:54,922 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:15:54,922 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:15:54,922 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:54,923 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47942, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:15:54,923 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b37d4d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:54,924 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45e8e518, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:54,924 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:15:54,924 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:15:54,924 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:15:54,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:15:54,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:15:54,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f0df7b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:54,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:15:54,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:15:54,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:54,925 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47958, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:15:54,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22add233, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:54,926 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10ff7be5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:54,926 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:15:54,926 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:15:54,926 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:15:54,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:15:54,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:15:54,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22e27419, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:54,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:15:54,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:15:54,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:15:54,927 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47974, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:15:54,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@447c0c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:15:54,928 DEBUG [Thread-1935 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:15:54,928 DEBUG [Thread-1937 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:15:54,928 DEBUG [Thread-1952 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:15:54,928 DEBUG [Thread-1944 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:15:54,928 DEBUG [Thread-1950 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:15:54,929 DEBUG [Thread-1939 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:15:54,929 DEBUG [Thread-1941 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:15:54,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:15:54,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:15:54,929 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:15:54,929 DEBUG [Thread-1946 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:15:54,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:15:54,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:15:54,929 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:15:54,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:15:54,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:15:54,929 DEBUG [Thread-1933 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:15:54,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:15:54,930 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:15:54,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:15:54,930 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:15:54,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:15:54,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:15:54,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:15:54,931 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:15:54,931 DEBUG [Thread-1948 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:15:54,931 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:15:54,931 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59496, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:15:54,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:15:54,931 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59508, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:15:54,932 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:15:54,932 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59514, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:15:54,932 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59516, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:15:54,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-14T09:15:54,932 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59528, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:15:54,932 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59540, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:15:54,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-14T09:15:54,932 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59562, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:15:54,932 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59578, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:15:54,933 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:15:54,933 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:15:54,933 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59556, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:15:54,933 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:15:54,933 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59582, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:15:54,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2] 2024-11-14T09:15:54,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2] 2024-11-14T09:15:54,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2] 2024-11-14T09:15:54,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2] 2024-11-14T09:15:54,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2] 2024-11-14T09:15:54,935 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2] 2024-11-14T09:15:54,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2] 2024-11-14T09:15:54,936 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:15:54,936 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2] 2024-11-14T09:15:54,936 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:15:54,937 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2] 2024-11-14T09:15:54,938 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2] 2024-11-14T09:15:54,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on add92018c22d01dad02f6ba9a2020355 2024-11-14T09:15:54,944 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing add92018c22d01dad02f6ba9a2020355 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T09:15:54,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK add92018c22d01dad02f6ba9a2020355, store=A 2024-11-14T09:15:54,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:15:54,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK add92018c22d01dad02f6ba9a2020355, store=B 2024-11-14T09:15:54,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:15:54,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK add92018c22d01dad02f6ba9a2020355, store=C 2024-11-14T09:15:54,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:15:54,964 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/ba37d9ac90774ee29895aaada4a4e3fa is 50, key is test_row_0/A:col10/1731575754937/Put/seqid=0 2024-11-14T09:15:54,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742253_1429 (size=12001) 2024-11-14T09:15:54,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731575754968, max timestamp=1731575754984Num uniques 859; 2024-11-14T09:15:54,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-14T09:15:54,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731575754968, max timestamp=1731575754984Num uniques 859; 2024-11-14T09:15:54,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-14T09:15:54,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731575754968, max timestamp=1731575754984Num uniques 859; 2024-11-14T09:15:54,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-14T09:15:55,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:55,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731575815020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:55,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:55,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:55,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:55,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-14T09:15:55,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:55,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1731575815021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:55,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:55,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:55,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:55,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59508 deadline: 1731575815024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:55,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:55,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59528 deadline: 1731575815023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:55,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:55,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:55,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59578 deadline: 1731575815024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:55,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:55,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:55,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:55,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:55,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:55,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:55,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:55,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:55,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:55,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-14T09:15:55,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:15:55,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. as already flushing 2024-11-14T09:15:55,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:15:55,090 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:55,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:55,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:55,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:55,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731575815135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:55,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:55,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:55,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:55,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:55,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1731575815145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:55,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:55,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:55,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59528 deadline: 1731575815146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:55,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:55,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:55,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:55,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:55,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59508 deadline: 1731575815146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:55,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:55,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:55,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:55,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:55,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59578 deadline: 1731575815146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:55,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:55,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:55,152 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:55,152 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:55,152 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:55,243 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-14T09:15:55,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:15:55,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. as already flushing 2024-11-14T09:15:55,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:15:55,243 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:55,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:55,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:55,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-14T09:15:55,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:55,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:55,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731575815354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:55,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1731575815355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:55,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:55,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:55,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:55,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:55,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59508 deadline: 1731575815355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:55,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:55,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:55,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:55,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:55,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:55,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:55,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:55,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59528 deadline: 1731575815355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:55,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:55,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:55,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:55,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59578 deadline: 1731575815355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:55,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:55,366 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:55,366 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:55,366 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:55,374 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/ba37d9ac90774ee29895aaada4a4e3fa 2024-11-14T09:15:55,394 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/d817f3936ba1404f88da0f57fbdb9818 is 50, key is test_row_0/B:col10/1731575754937/Put/seqid=0 2024-11-14T09:15:55,396 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-14T09:15:55,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:15:55,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. as already flushing 2024-11-14T09:15:55,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:15:55,396 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:55,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:55,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:55,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742254_1430 (size=12001) 2024-11-14T09:15:55,549 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-14T09:15:55,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:15:55,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. as already flushing 2024-11-14T09:15:55,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:15:55,549 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:55,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:55,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:55,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-14T09:15:55,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:55,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731575815675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:55,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:55,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1731575815675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:55,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:55,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:55,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:55,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:55,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59508 deadline: 1731575815675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:55,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:55,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:55,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:55,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:55,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:55,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:55,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59528 deadline: 1731575815675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:55,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:55,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:55,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:55,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:55,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59578 deadline: 1731575815675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:55,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:55,685 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:55,685 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:55,685 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:55,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-14T09:15:55,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:15:55,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. as already flushing 2024-11-14T09:15:55,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:15:55,702 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:55,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:55,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:55,798 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/d817f3936ba1404f88da0f57fbdb9818 2024-11-14T09:15:55,820 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/ee992efded7442eda59e0967421678df is 50, key is test_row_0/C:col10/1731575754937/Put/seqid=0 2024-11-14T09:15:55,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742255_1431 (size=12001) 2024-11-14T09:15:55,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-14T09:15:55,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:15:55,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. as already flushing 2024-11-14T09:15:55,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:15:55,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:55,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:55,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:56,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-14T09:15:56,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:15:56,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. as already flushing 2024-11-14T09:15:56,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:15:56,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:56,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:56,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:56,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-14T09:15:56,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-14T09:15:56,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:15:56,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. as already flushing 2024-11-14T09:15:56,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:15:56,161 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:56,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:56,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:15:56,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:56,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731575816195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:56,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:56,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:56,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:56,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:56,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1731575816195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:56,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:56,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:56,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:56,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:56,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59508 deadline: 1731575816195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:56,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:56,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:56,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:56,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:56,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59528 deadline: 1731575816195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:56,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:56,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:56,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59578 deadline: 1731575816195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:56,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:56,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:56,206 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:56,206 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:56,206 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:56,225 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/ee992efded7442eda59e0967421678df 2024-11-14T09:15:56,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/ba37d9ac90774ee29895aaada4a4e3fa as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/ba37d9ac90774ee29895aaada4a4e3fa 2024-11-14T09:15:56,231 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/ba37d9ac90774ee29895aaada4a4e3fa, entries=150, sequenceid=12, filesize=11.7 K 2024-11-14T09:15:56,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/d817f3936ba1404f88da0f57fbdb9818 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/d817f3936ba1404f88da0f57fbdb9818 2024-11-14T09:15:56,235 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/d817f3936ba1404f88da0f57fbdb9818, entries=150, sequenceid=12, filesize=11.7 K 2024-11-14T09:15:56,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/ee992efded7442eda59e0967421678df as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/ee992efded7442eda59e0967421678df 2024-11-14T09:15:56,238 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/ee992efded7442eda59e0967421678df, entries=150, sequenceid=12, filesize=11.7 K 2024-11-14T09:15:56,239 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=140.89 KB/144270 for add92018c22d01dad02f6ba9a2020355 in 1296ms, sequenceid=12, compaction requested=false 2024-11-14T09:15:56,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for add92018c22d01dad02f6ba9a2020355: 2024-11-14T09:15:56,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-14T09:15:56,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:15:56,314 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing add92018c22d01dad02f6ba9a2020355 3/3 column families, dataSize=140.89 KB heapSize=390.84 KB 2024-11-14T09:15:56,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK add92018c22d01dad02f6ba9a2020355, store=A 2024-11-14T09:15:56,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:15:56,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK add92018c22d01dad02f6ba9a2020355, store=B 2024-11-14T09:15:56,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:15:56,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK add92018c22d01dad02f6ba9a2020355, store=C 2024-11-14T09:15:56,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:15:56,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/26bf1c95ed1b49e8b45b39fc20248a18 is 50, key is test_row_0/A:col10/1731575755003/Put/seqid=0 2024-11-14T09:15:56,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742256_1432 (size=12001) 2024-11-14T09:15:56,723 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/26bf1c95ed1b49e8b45b39fc20248a18 2024-11-14T09:15:56,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/b7b74dc119ef473db227fc4219651f94 is 50, key is test_row_0/B:col10/1731575755003/Put/seqid=0 2024-11-14T09:15:56,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742257_1433 (size=12001) 2024-11-14T09:15:57,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-14T09:15:57,135 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/b7b74dc119ef473db227fc4219651f94 2024-11-14T09:15:57,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/8e9825e6700d4756ac9bceb4b1766bc2 is 50, key is test_row_0/C:col10/1731575755003/Put/seqid=0 2024-11-14T09:15:57,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742258_1434 (size=12001) 2024-11-14T09:15:57,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on add92018c22d01dad02f6ba9a2020355 2024-11-14T09:15:57,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. as already flushing 2024-11-14T09:15:57,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:57,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1731575817227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:57,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:57,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59508 deadline: 1731575817238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:57,240 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:57,240 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:57,240 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:57,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:57,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:57,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:57,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:57,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:57,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59578 deadline: 1731575817240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:57,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59528 deadline: 1731575817240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:57,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:57,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:57,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731575817240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:57,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:57,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:57,253 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:57,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:57,253 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:57,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:57,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:57,253 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:57,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:57,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1731575817344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:57,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:57,354 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:57,354 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:57,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59508 deadline: 1731575817345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:57,355 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:57,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:57,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:57,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:57,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:57,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59528 deadline: 1731575817355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:57,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:57,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731575817355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:57,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:57,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:57,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:57,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:57,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:57,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59578 deadline: 1731575817355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:57,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:57,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:57,360 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:57,360 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:57,360 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:57,547 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/8e9825e6700d4756ac9bceb4b1766bc2 2024-11-14T09:15:57,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/26bf1c95ed1b49e8b45b39fc20248a18 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/26bf1c95ed1b49e8b45b39fc20248a18 2024-11-14T09:15:57,554 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/26bf1c95ed1b49e8b45b39fc20248a18, entries=150, sequenceid=47, filesize=11.7 K 2024-11-14T09:15:57,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/b7b74dc119ef473db227fc4219651f94 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/b7b74dc119ef473db227fc4219651f94 2024-11-14T09:15:57,558 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/b7b74dc119ef473db227fc4219651f94, entries=150, sequenceid=47, filesize=11.7 K 2024-11-14T09:15:57,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/8e9825e6700d4756ac9bceb4b1766bc2 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/8e9825e6700d4756ac9bceb4b1766bc2 2024-11-14T09:15:57,561 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/8e9825e6700d4756ac9bceb4b1766bc2, entries=150, sequenceid=47, filesize=11.7 K 2024-11-14T09:15:57,562 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~390.80 KB/400176, currentSize=60.38 KB/61830 for add92018c22d01dad02f6ba9a2020355 in 1248ms, sequenceid=47, compaction requested=false 2024-11-14T09:15:57,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for add92018c22d01dad02f6ba9a2020355: 2024-11-14T09:15:57,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:15:57,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-14T09:15:57,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-14T09:15:57,567 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-14T09:15:57,567 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6280 sec 2024-11-14T09:15:57,568 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 2.6370 sec 2024-11-14T09:15:57,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on add92018c22d01dad02f6ba9a2020355 2024-11-14T09:15:57,569 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing add92018c22d01dad02f6ba9a2020355 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-14T09:15:57,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK add92018c22d01dad02f6ba9a2020355, store=A 2024-11-14T09:15:57,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:15:57,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK add92018c22d01dad02f6ba9a2020355, store=B 2024-11-14T09:15:57,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:15:57,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK add92018c22d01dad02f6ba9a2020355, store=C 2024-11-14T09:15:57,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:15:57,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/debed659e8f54619937a415a7d17d7e4 is 50, key is test_row_0/A:col10/1731575757569/Put/seqid=0 2024-11-14T09:15:57,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742259_1435 (size=12001) 2024-11-14T09:15:57,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:57,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59528 deadline: 1731575817590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:57,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:57,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:57,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59578 deadline: 1731575817591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:57,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:57,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:57,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:57,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731575817592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:57,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:57,606 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:57,606 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:57,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:57,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:57,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:57,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:57,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59508 deadline: 1731575817605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:57,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:57,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1731575817605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:57,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:57,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:57,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:57,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:57,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:57,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:57,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:57,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59528 deadline: 1731575817715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:57,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:57,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:57,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:57,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59578 deadline: 1731575817715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:57,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:57,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:57,726 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:57,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731575817715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:57,726 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:57,726 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:57,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:57,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:57,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:57,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:57,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59508 deadline: 1731575817716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:57,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:57,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:57,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:57,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:57,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1731575817725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:57,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:57,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:57,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:57,801 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T09:15:57,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:57,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59528 deadline: 1731575817934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:57,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:57,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:57,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:57,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59578 deadline: 1731575817934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:57,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:57,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:57,944 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:57,944 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:57,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731575817934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:57,944 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:57,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:57,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:57,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59508 deadline: 1731575817935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:57,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:57,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:57,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:57,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:57,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:57,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:57,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1731575817945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:57,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:57,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:57,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:57,977 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/debed659e8f54619937a415a7d17d7e4 2024-11-14T09:15:57,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/3be4aa6b98bb412fb38eaa8c92a0b02f is 50, key is test_row_0/B:col10/1731575757569/Put/seqid=0 2024-11-14T09:15:57,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742260_1436 (size=12001) 2024-11-14T09:15:58,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:58,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59528 deadline: 1731575818255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:58,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:58,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59578 deadline: 1731575818255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:58,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:58,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:58,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:58,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:58,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731575818255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:58,262 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:58,262 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:58,262 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:58,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:58,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:58,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59508 deadline: 1731575818255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:58,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:58,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:58,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:58,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:58,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:58,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:58,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1731575818265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:58,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:58,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:58,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:58,388 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/3be4aa6b98bb412fb38eaa8c92a0b02f 2024-11-14T09:15:58,394 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/30707c155e5a4e33bf486a51a99167f0 is 50, key is test_row_0/C:col10/1731575757569/Put/seqid=0 2024-11-14T09:15:58,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742261_1437 (size=12001) 2024-11-14T09:15:58,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:58,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59528 deadline: 1731575818765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:58,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:58,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:58,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:58,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59578 deadline: 1731575818765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:58,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:58,775 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:58,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:58,775 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:58,775 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:58,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731575818765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:58,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:58,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:58,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:58,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:58,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59508 deadline: 1731575818766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:58,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:58,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:58,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:58,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:58,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1731575818785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:58,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:58,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:58,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:58,798 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/30707c155e5a4e33bf486a51a99167f0 2024-11-14T09:15:58,802 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/debed659e8f54619937a415a7d17d7e4 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/debed659e8f54619937a415a7d17d7e4 2024-11-14T09:15:58,805 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/debed659e8f54619937a415a7d17d7e4, entries=150, sequenceid=61, filesize=11.7 K 2024-11-14T09:15:58,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/3be4aa6b98bb412fb38eaa8c92a0b02f as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/3be4aa6b98bb412fb38eaa8c92a0b02f 2024-11-14T09:15:58,809 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/3be4aa6b98bb412fb38eaa8c92a0b02f, entries=150, sequenceid=61, filesize=11.7 K 2024-11-14T09:15:58,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/30707c155e5a4e33bf486a51a99167f0 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/30707c155e5a4e33bf486a51a99167f0 2024-11-14T09:15:58,812 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/30707c155e5a4e33bf486a51a99167f0, entries=150, sequenceid=61, filesize=11.7 K 2024-11-14T09:15:58,813 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for add92018c22d01dad02f6ba9a2020355 in 1244ms, sequenceid=61, compaction requested=true 2024-11-14T09:15:58,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for add92018c22d01dad02f6ba9a2020355: 2024-11-14T09:15:58,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store add92018c22d01dad02f6ba9a2020355:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:15:58,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:15:58,814 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:15:58,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store add92018c22d01dad02f6ba9a2020355:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:15:58,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:15:58,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store add92018c22d01dad02f6ba9a2020355:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:15:58,814 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:15:58,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:15:58,815 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:15:58,815 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:15:58,815 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): add92018c22d01dad02f6ba9a2020355/A is initiating minor compaction (all files) 2024-11-14T09:15:58,815 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): add92018c22d01dad02f6ba9a2020355/B is initiating minor compaction (all files) 2024-11-14T09:15:58,815 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of add92018c22d01dad02f6ba9a2020355/A in TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:15:58,815 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of add92018c22d01dad02f6ba9a2020355/B in TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:15:58,815 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/ba37d9ac90774ee29895aaada4a4e3fa, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/26bf1c95ed1b49e8b45b39fc20248a18, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/debed659e8f54619937a415a7d17d7e4] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp, totalSize=35.2 K 2024-11-14T09:15:58,815 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/d817f3936ba1404f88da0f57fbdb9818, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/b7b74dc119ef473db227fc4219651f94, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/3be4aa6b98bb412fb38eaa8c92a0b02f] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp, totalSize=35.2 K 2024-11-14T09:15:58,815 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba37d9ac90774ee29895aaada4a4e3fa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731575754937 2024-11-14T09:15:58,815 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting d817f3936ba1404f88da0f57fbdb9818, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731575754937 2024-11-14T09:15:58,816 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26bf1c95ed1b49e8b45b39fc20248a18, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731575755003 2024-11-14T09:15:58,816 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting b7b74dc119ef473db227fc4219651f94, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731575755003 2024-11-14T09:15:58,816 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting debed659e8f54619937a415a7d17d7e4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731575757238 2024-11-14T09:15:58,816 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 3be4aa6b98bb412fb38eaa8c92a0b02f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731575757238 2024-11-14T09:15:58,823 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): add92018c22d01dad02f6ba9a2020355#B#compaction#367 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:15:58,823 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): add92018c22d01dad02f6ba9a2020355#A#compaction#366 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:15:58,823 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/990b7d3155354ea1bfa11fd705c525aa is 50, key is test_row_0/B:col10/1731575757569/Put/seqid=0 2024-11-14T09:15:58,823 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/02135b1ac4e74ee48e3b9d0d841131c6 is 50, key is test_row_0/A:col10/1731575757569/Put/seqid=0 2024-11-14T09:15:58,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742263_1439 (size=12104) 2024-11-14T09:15:58,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742262_1438 (size=12104) 2024-11-14T09:15:58,847 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/02135b1ac4e74ee48e3b9d0d841131c6 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/02135b1ac4e74ee48e3b9d0d841131c6 2024-11-14T09:15:58,851 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in add92018c22d01dad02f6ba9a2020355/A of add92018c22d01dad02f6ba9a2020355 into 02135b1ac4e74ee48e3b9d0d841131c6(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:15:58,851 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for add92018c22d01dad02f6ba9a2020355: 2024-11-14T09:15:58,851 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., storeName=add92018c22d01dad02f6ba9a2020355/A, priority=13, startTime=1731575758813; duration=0sec 2024-11-14T09:15:58,851 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:15:58,851 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: add92018c22d01dad02f6ba9a2020355:A 2024-11-14T09:15:58,851 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:15:58,852 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:15:58,852 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): add92018c22d01dad02f6ba9a2020355/C is initiating minor compaction (all files) 2024-11-14T09:15:58,852 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of add92018c22d01dad02f6ba9a2020355/C in TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:15:58,852 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/ee992efded7442eda59e0967421678df, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/8e9825e6700d4756ac9bceb4b1766bc2, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/30707c155e5a4e33bf486a51a99167f0] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp, totalSize=35.2 K 2024-11-14T09:15:58,853 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee992efded7442eda59e0967421678df, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731575754937 2024-11-14T09:15:58,853 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e9825e6700d4756ac9bceb4b1766bc2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731575755003 2024-11-14T09:15:58,853 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30707c155e5a4e33bf486a51a99167f0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731575757238 2024-11-14T09:15:58,859 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): add92018c22d01dad02f6ba9a2020355#C#compaction#368 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:15:58,859 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/90a828f277c24dfcbe91b7dc2171743b is 50, key is test_row_0/C:col10/1731575757569/Put/seqid=0 2024-11-14T09:15:58,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742264_1440 (size=12104) 2024-11-14T09:15:58,869 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/90a828f277c24dfcbe91b7dc2171743b as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/90a828f277c24dfcbe91b7dc2171743b 2024-11-14T09:15:58,874 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in add92018c22d01dad02f6ba9a2020355/C of add92018c22d01dad02f6ba9a2020355 into 90a828f277c24dfcbe91b7dc2171743b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:15:58,874 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for add92018c22d01dad02f6ba9a2020355: 2024-11-14T09:15:58,874 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., storeName=add92018c22d01dad02f6ba9a2020355/C, priority=13, startTime=1731575758814; duration=0sec 2024-11-14T09:15:58,874 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:15:58,874 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: add92018c22d01dad02f6ba9a2020355:C 2024-11-14T09:15:59,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-14T09:15:59,096 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:15:59,096 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:15:59,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-14T09:15:59,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T09:15:59,098 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:15:59,098 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:15:59,099 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:15:59,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T09:15:59,236 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/990b7d3155354ea1bfa11fd705c525aa as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/990b7d3155354ea1bfa11fd705c525aa 2024-11-14T09:15:59,240 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in add92018c22d01dad02f6ba9a2020355/B of add92018c22d01dad02f6ba9a2020355 into 990b7d3155354ea1bfa11fd705c525aa(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:15:59,240 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for add92018c22d01dad02f6ba9a2020355: 2024-11-14T09:15:59,240 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., storeName=add92018c22d01dad02f6ba9a2020355/B, priority=13, startTime=1731575758814; duration=0sec 2024-11-14T09:15:59,240 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:15:59,240 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: add92018c22d01dad02f6ba9a2020355:B 2024-11-14T09:15:59,250 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-14T09:15:59,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:15:59,251 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing add92018c22d01dad02f6ba9a2020355 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-14T09:15:59,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK add92018c22d01dad02f6ba9a2020355, store=A 2024-11-14T09:15:59,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:15:59,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK add92018c22d01dad02f6ba9a2020355, store=B 2024-11-14T09:15:59,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:15:59,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK add92018c22d01dad02f6ba9a2020355, store=C 2024-11-14T09:15:59,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:15:59,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/2188ece040b347a49101b9b2dfa4f70a is 50, key is test_row_0/A:col10/1731575757603/Put/seqid=0 2024-11-14T09:15:59,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742265_1441 (size=12001) 2024-11-14T09:15:59,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T09:15:59,661 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/2188ece040b347a49101b9b2dfa4f70a 2024-11-14T09:15:59,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/d7e6c10f2cbe445392c0a4b5e08eef7e is 50, key is test_row_0/B:col10/1731575757603/Put/seqid=0 2024-11-14T09:15:59,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742266_1442 (size=12001) 2024-11-14T09:15:59,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T09:15:59,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on add92018c22d01dad02f6ba9a2020355 2024-11-14T09:15:59,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. as already flushing 2024-11-14T09:15:59,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:59,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59578 deadline: 1731575819802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:59,807 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:59,807 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:59,807 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:59,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:59,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59508 deadline: 1731575819803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:59,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:59,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:59,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:59,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59528 deadline: 1731575819804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:59,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:59,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:59,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:59,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:59,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:59,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1731575819805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:59,814 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:59,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:59,814 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:59,814 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:59,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731575819806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:59,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:59,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:59,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:59,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:59,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59578 deadline: 1731575819915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:59,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:59,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59508 deadline: 1731575819915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:59,922 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:59,922 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:59,922 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:59,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:59,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59528 deadline: 1731575819915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:59,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:59,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:59,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:59,922 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:59,922 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:59,922 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:59,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:59,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1731575819925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:59,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:15:59,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:59,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731575819925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:15:59,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:59,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:15:59,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:59,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:15:59,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:00,078 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/d7e6c10f2cbe445392c0a4b5e08eef7e 2024-11-14T09:16:00,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/27cb1bd010f24e04a0a674d60208953c is 50, key is test_row_0/C:col10/1731575757603/Put/seqid=0 2024-11-14T09:16:00,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742267_1443 (size=12001) 2024-11-14T09:16:00,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:00,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59578 deadline: 1731575820124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:00,133 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:00,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:00,134 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:00,134 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:00,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59508 deadline: 1731575820124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:00,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:00,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:00,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:00,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:00,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59528 deadline: 1731575820125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:00,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:00,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:00,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:00,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:00,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1731575820135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:00,141 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:00,141 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:00,141 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:00,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:00,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731575820145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:00,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:00,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:00,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:00,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T09:16:00,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:00,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59578 deadline: 1731575820444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:00,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:00,451 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:00,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59508 deadline: 1731575820444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:00,451 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:00,451 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:00,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:00,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:00,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:00,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:00,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59528 deadline: 1731575820444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:00,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:00,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:00,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:00,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:00,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1731575820455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:00,462 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:00,462 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:00,462 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:00,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:00,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731575820465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:00,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:00,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:00,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:00,493 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/27cb1bd010f24e04a0a674d60208953c 2024-11-14T09:16:00,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/2188ece040b347a49101b9b2dfa4f70a as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/2188ece040b347a49101b9b2dfa4f70a 2024-11-14T09:16:00,501 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/2188ece040b347a49101b9b2dfa4f70a, entries=150, sequenceid=82, filesize=11.7 K 2024-11-14T09:16:00,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/d7e6c10f2cbe445392c0a4b5e08eef7e as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/d7e6c10f2cbe445392c0a4b5e08eef7e 2024-11-14T09:16:00,505 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/d7e6c10f2cbe445392c0a4b5e08eef7e, entries=150, sequenceid=82, filesize=11.7 K 2024-11-14T09:16:00,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/27cb1bd010f24e04a0a674d60208953c as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/27cb1bd010f24e04a0a674d60208953c 2024-11-14T09:16:00,508 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/27cb1bd010f24e04a0a674d60208953c, entries=150, sequenceid=82, filesize=11.7 K 2024-11-14T09:16:00,509 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for add92018c22d01dad02f6ba9a2020355 in 1258ms, sequenceid=82, compaction requested=false 2024-11-14T09:16:00,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for add92018c22d01dad02f6ba9a2020355: 2024-11-14T09:16:00,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:00,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-14T09:16:00,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-14T09:16:00,512 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-14T09:16:00,513 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4120 sec 2024-11-14T09:16:00,514 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.4170 sec 2024-11-14T09:16:00,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on add92018c22d01dad02f6ba9a2020355 2024-11-14T09:16:00,960 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing add92018c22d01dad02f6ba9a2020355 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-14T09:16:00,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK add92018c22d01dad02f6ba9a2020355, store=A 2024-11-14T09:16:00,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:00,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK add92018c22d01dad02f6ba9a2020355, store=B 2024-11-14T09:16:00,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:00,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK add92018c22d01dad02f6ba9a2020355, store=C 2024-11-14T09:16:00,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:00,965 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/1001a22b2ff04723bd6f6f18e0e12002 is 50, key is test_row_0/A:col10/1731575759803/Put/seqid=0 2024-11-14T09:16:00,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742268_1444 (size=11997) 2024-11-14T09:16:00,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:00,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1731575820984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:00,993 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:00,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:00,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:00,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:00,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731575820985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:00,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:00,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:00,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:00,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:00,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59528 deadline: 1731575820985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:00,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:00,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:00,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:00,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:00,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59508 deadline: 1731575820992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:00,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:00,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:00,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:01,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:01,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59578 deadline: 1731575820993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:01,004 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:01,004 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:01,004 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:01,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:01,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1731575821105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:01,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:01,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:01,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:01,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59528 deadline: 1731575821105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:01,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:01,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:01,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:01,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:01,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:01,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59508 deadline: 1731575821105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:01,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:01,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:01,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:01,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:01,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59578 deadline: 1731575821115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:01,119 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:01,119 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:01,119 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:01,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T09:16:01,245 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:16:01,246 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:16:01,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-14T09:16:01,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-14T09:16:01,248 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:16:01,249 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:16:01,249 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:16:01,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:01,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1731575821314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:01,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:01,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:01,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:01,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59528 deadline: 1731575821314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:01,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:01,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:01,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59508 deadline: 1731575821315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:01,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:01,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:01,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:01,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:01,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:01,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:01,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:01,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59578 deadline: 1731575821325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:01,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:01,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:01,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:01,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-14T09:16:01,369 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/1001a22b2ff04723bd6f6f18e0e12002 2024-11-14T09:16:01,376 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/055b51dd4ab24453a712599caf9e8e06 is 50, key is test_row_0/B:col10/1731575759803/Put/seqid=0 2024-11-14T09:16:01,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742269_1445 (size=9657) 2024-11-14T09:16:01,380 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/055b51dd4ab24453a712599caf9e8e06 2024-11-14T09:16:01,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/b5c24bad483d43bfabbb5e283f17c7c3 is 50, key is test_row_0/C:col10/1731575759803/Put/seqid=0 2024-11-14T09:16:01,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742270_1446 (size=9657) 2024-11-14T09:16:01,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-14T09:16:01,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:01,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. as already flushing 2024-11-14T09:16:01,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:01,401 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:01,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:01,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:01,554 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-14T09:16:01,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:01,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. as already flushing 2024-11-14T09:16:01,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:01,554 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:01,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:01,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:01,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-14T09:16:01,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:01,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1731575821625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:01,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:01,629 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:01,629 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:01,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59528 deadline: 1731575821625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:01,629 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:01,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:01,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:01,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:01,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:01,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59508 deadline: 1731575821625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:01,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:01,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:01,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:01,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:01,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59578 deadline: 1731575821635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:01,641 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:01,641 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:01,641 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:01,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-14T09:16:01,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:01,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. as already flushing 2024-11-14T09:16:01,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:01,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:01,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:01,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:01,790 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/b5c24bad483d43bfabbb5e283f17c7c3 2024-11-14T09:16:01,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/1001a22b2ff04723bd6f6f18e0e12002 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/1001a22b2ff04723bd6f6f18e0e12002 2024-11-14T09:16:01,796 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/1001a22b2ff04723bd6f6f18e0e12002, entries=150, sequenceid=97, filesize=11.7 K 2024-11-14T09:16:01,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/055b51dd4ab24453a712599caf9e8e06 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/055b51dd4ab24453a712599caf9e8e06 2024-11-14T09:16:01,800 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/055b51dd4ab24453a712599caf9e8e06, entries=100, sequenceid=97, filesize=9.4 K 2024-11-14T09:16:01,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/b5c24bad483d43bfabbb5e283f17c7c3 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/b5c24bad483d43bfabbb5e283f17c7c3 2024-11-14T09:16:01,804 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/b5c24bad483d43bfabbb5e283f17c7c3, entries=100, sequenceid=97, filesize=9.4 K 2024-11-14T09:16:01,805 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for add92018c22d01dad02f6ba9a2020355 in 845ms, sequenceid=97, compaction requested=true 2024-11-14T09:16:01,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for add92018c22d01dad02f6ba9a2020355: 2024-11-14T09:16:01,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store add92018c22d01dad02f6ba9a2020355:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:16:01,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:16:01,805 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:16:01,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store add92018c22d01dad02f6ba9a2020355:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:16:01,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:16:01,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store add92018c22d01dad02f6ba9a2020355:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:16:01,806 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:16:01,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:16:01,806 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36102 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:16:01,806 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): add92018c22d01dad02f6ba9a2020355/A is initiating minor compaction (all files) 2024-11-14T09:16:01,807 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of add92018c22d01dad02f6ba9a2020355/A in TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:01,807 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/02135b1ac4e74ee48e3b9d0d841131c6, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/2188ece040b347a49101b9b2dfa4f70a, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/1001a22b2ff04723bd6f6f18e0e12002] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp, totalSize=35.3 K 2024-11-14T09:16:01,807 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:16:01,807 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): add92018c22d01dad02f6ba9a2020355/B is initiating minor compaction (all files) 2024-11-14T09:16:01,807 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of add92018c22d01dad02f6ba9a2020355/B in TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:01,807 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/990b7d3155354ea1bfa11fd705c525aa, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/d7e6c10f2cbe445392c0a4b5e08eef7e, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/055b51dd4ab24453a712599caf9e8e06] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp, totalSize=33.0 K 2024-11-14T09:16:01,807 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02135b1ac4e74ee48e3b9d0d841131c6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731575757238 2024-11-14T09:16:01,808 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 990b7d3155354ea1bfa11fd705c525aa, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731575757238 2024-11-14T09:16:01,808 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2188ece040b347a49101b9b2dfa4f70a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731575757590 2024-11-14T09:16:01,808 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting d7e6c10f2cbe445392c0a4b5e08eef7e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731575757590 2024-11-14T09:16:01,808 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1001a22b2ff04723bd6f6f18e0e12002, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731575759803 2024-11-14T09:16:01,808 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 055b51dd4ab24453a712599caf9e8e06, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731575759803 2024-11-14T09:16:01,823 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): add92018c22d01dad02f6ba9a2020355#B#compaction#375 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:16:01,824 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/aae8d9e61eed42d9ba3a4476344ef224 is 50, key is test_row_0/B:col10/1731575759803/Put/seqid=0 2024-11-14T09:16:01,826 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): add92018c22d01dad02f6ba9a2020355#A#compaction#376 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:16:01,826 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/030bf1d59aa642fe9bedc8cf989fb5de is 50, key is test_row_0/A:col10/1731575759803/Put/seqid=0 2024-11-14T09:16:01,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742271_1447 (size=12207) 2024-11-14T09:16:01,836 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/aae8d9e61eed42d9ba3a4476344ef224 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/aae8d9e61eed42d9ba3a4476344ef224 2024-11-14T09:16:01,840 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in add92018c22d01dad02f6ba9a2020355/B of add92018c22d01dad02f6ba9a2020355 into aae8d9e61eed42d9ba3a4476344ef224(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:16:01,840 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for add92018c22d01dad02f6ba9a2020355: 2024-11-14T09:16:01,840 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., storeName=add92018c22d01dad02f6ba9a2020355/B, priority=13, startTime=1731575761805; duration=0sec 2024-11-14T09:16:01,840 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:16:01,840 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: add92018c22d01dad02f6ba9a2020355:B 2024-11-14T09:16:01,840 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:16:01,841 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:16:01,841 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): add92018c22d01dad02f6ba9a2020355/C is initiating minor compaction (all files) 2024-11-14T09:16:01,842 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of add92018c22d01dad02f6ba9a2020355/C in TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:01,842 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/90a828f277c24dfcbe91b7dc2171743b, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/27cb1bd010f24e04a0a674d60208953c, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/b5c24bad483d43bfabbb5e283f17c7c3] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp, totalSize=33.0 K 2024-11-14T09:16:01,842 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 90a828f277c24dfcbe91b7dc2171743b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731575757238 2024-11-14T09:16:01,842 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 27cb1bd010f24e04a0a674d60208953c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731575757590 2024-11-14T09:16:01,842 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting b5c24bad483d43bfabbb5e283f17c7c3, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731575759803 2024-11-14T09:16:01,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742272_1448 (size=12207) 2024-11-14T09:16:01,850 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): add92018c22d01dad02f6ba9a2020355#C#compaction#377 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:16:01,851 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/030bf1d59aa642fe9bedc8cf989fb5de as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/030bf1d59aa642fe9bedc8cf989fb5de 2024-11-14T09:16:01,851 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/d34eabbeb03042f986c596343aa45e5b is 50, key is test_row_0/C:col10/1731575759803/Put/seqid=0 2024-11-14T09:16:01,858 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in add92018c22d01dad02f6ba9a2020355/A of add92018c22d01dad02f6ba9a2020355 into 030bf1d59aa642fe9bedc8cf989fb5de(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:16:01,858 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for add92018c22d01dad02f6ba9a2020355: 2024-11-14T09:16:01,858 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., storeName=add92018c22d01dad02f6ba9a2020355/A, priority=13, startTime=1731575761805; duration=0sec 2024-11-14T09:16:01,858 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:16:01,858 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: add92018c22d01dad02f6ba9a2020355:A 2024-11-14T09:16:01,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742273_1449 (size=12207) 2024-11-14T09:16:01,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-14T09:16:01,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:01,860 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing add92018c22d01dad02f6ba9a2020355 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-14T09:16:01,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK add92018c22d01dad02f6ba9a2020355, store=A 2024-11-14T09:16:01,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:01,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK add92018c22d01dad02f6ba9a2020355, store=B 2024-11-14T09:16:01,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:01,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK add92018c22d01dad02f6ba9a2020355, store=C 2024-11-14T09:16:01,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:01,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/554e492687144122a63953e9af68a4cc is 50, key is test_row_0/A:col10/1731575760992/Put/seqid=0 2024-11-14T09:16:01,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742274_1450 (size=12001) 2024-11-14T09:16:01,870 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/554e492687144122a63953e9af68a4cc 2024-11-14T09:16:01,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-14T09:16:01,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/f61bf08d6f4f419c9cd0eb400f1b64af is 50, key is test_row_0/B:col10/1731575760992/Put/seqid=0 2024-11-14T09:16:01,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742275_1451 (size=12001) 2024-11-14T09:16:02,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on add92018c22d01dad02f6ba9a2020355 2024-11-14T09:16:02,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. as already flushing 2024-11-14T09:16:02,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:02,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731575822078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:02,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:02,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:02,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:02,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:02,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59528 deadline: 1731575822135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:02,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:02,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:02,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:02,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:02,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59508 deadline: 1731575822135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:02,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:02,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:02,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:02,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:02,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1731575822145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:02,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:02,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:02,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:02,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:02,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59578 deadline: 1731575822154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:02,162 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:02,163 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:02,163 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:02,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:02,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731575822185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:02,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:02,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:02,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:02,263 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/d34eabbeb03042f986c596343aa45e5b as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/d34eabbeb03042f986c596343aa45e5b 2024-11-14T09:16:02,267 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in add92018c22d01dad02f6ba9a2020355/C of add92018c22d01dad02f6ba9a2020355 into d34eabbeb03042f986c596343aa45e5b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:16:02,267 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for add92018c22d01dad02f6ba9a2020355: 2024-11-14T09:16:02,267 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., storeName=add92018c22d01dad02f6ba9a2020355/C, priority=13, startTime=1731575761805; duration=0sec 2024-11-14T09:16:02,267 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:16:02,267 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: add92018c22d01dad02f6ba9a2020355:C 2024-11-14T09:16:02,286 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/f61bf08d6f4f419c9cd0eb400f1b64af 2024-11-14T09:16:02,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/928e0de62abb4c18b9a9eb412ea40a44 is 50, key is test_row_0/C:col10/1731575760992/Put/seqid=0 2024-11-14T09:16:02,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742276_1452 (size=12001) 2024-11-14T09:16:02,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-14T09:16:02,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:02,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731575822405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:02,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:02,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:02,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:02,698 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/928e0de62abb4c18b9a9eb412ea40a44 2024-11-14T09:16:02,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/554e492687144122a63953e9af68a4cc as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/554e492687144122a63953e9af68a4cc 2024-11-14T09:16:02,705 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/554e492687144122a63953e9af68a4cc, entries=150, sequenceid=116, filesize=11.7 K 2024-11-14T09:16:02,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/f61bf08d6f4f419c9cd0eb400f1b64af as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/f61bf08d6f4f419c9cd0eb400f1b64af 2024-11-14T09:16:02,709 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/f61bf08d6f4f419c9cd0eb400f1b64af, entries=150, sequenceid=116, filesize=11.7 K 2024-11-14T09:16:02,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/928e0de62abb4c18b9a9eb412ea40a44 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/928e0de62abb4c18b9a9eb412ea40a44 2024-11-14T09:16:02,713 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/928e0de62abb4c18b9a9eb412ea40a44, entries=150, sequenceid=116, filesize=11.7 K 2024-11-14T09:16:02,714 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for add92018c22d01dad02f6ba9a2020355 in 854ms, sequenceid=116, compaction requested=false 2024-11-14T09:16:02,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for add92018c22d01dad02f6ba9a2020355: 2024-11-14T09:16:02,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:02,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-14T09:16:02,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-14T09:16:02,717 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-14T09:16:02,717 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4660 sec 2024-11-14T09:16:02,719 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.4720 sec 2024-11-14T09:16:02,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on add92018c22d01dad02f6ba9a2020355 2024-11-14T09:16:02,732 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing add92018c22d01dad02f6ba9a2020355 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-14T09:16:02,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK add92018c22d01dad02f6ba9a2020355, store=A 2024-11-14T09:16:02,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:02,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK add92018c22d01dad02f6ba9a2020355, store=B 2024-11-14T09:16:02,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:02,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK add92018c22d01dad02f6ba9a2020355, store=C 2024-11-14T09:16:02,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:02,736 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/d1752cd7a73848dbae7b473c725e76a6 is 50, key is test_row_0/A:col10/1731575762065/Put/seqid=0 2024-11-14T09:16:02,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742277_1453 (size=14491) 2024-11-14T09:16:02,740 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/d1752cd7a73848dbae7b473c725e76a6 2024-11-14T09:16:02,758 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/3d040a1f59cc41b29d7911787c49a827 is 50, key is test_row_0/B:col10/1731575762065/Put/seqid=0 2024-11-14T09:16:02,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742278_1454 (size=12101) 2024-11-14T09:16:02,762 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/3d040a1f59cc41b29d7911787c49a827 2024-11-14T09:16:02,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/71a2849408e54180989bc1873890cc3b is 50, key is test_row_0/C:col10/1731575762065/Put/seqid=0 2024-11-14T09:16:02,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742279_1455 (size=12101) 2024-11-14T09:16:02,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:02,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731575822828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:02,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:02,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:02,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:02,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:02,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731575822945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:02,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:02,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:02,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:03,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:03,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59528 deadline: 1731575823155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:03,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:03,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59508 deadline: 1731575823155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:03,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:03,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731575823155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:03,159 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_1' in add92018c22d01dad02f6ba9a2020355 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:16:03,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:03,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:03,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:03,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:03,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:03,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:03,159 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_1' in add92018c22d01dad02f6ba9a2020355 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:16:03,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:03,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:03,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:03,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:03,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1731575823165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:03,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:03,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59578 deadline: 1731575823165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:03,172 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_1' in add92018c22d01dad02f6ba9a2020355 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:16:03,173 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:03,173 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:03,173 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:03,172 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_2' in add92018c22d01dad02f6ba9a2020355 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:16:03,173 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:03,173 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:03,173 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:03,175 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/71a2849408e54180989bc1873890cc3b 2024-11-14T09:16:03,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/d1752cd7a73848dbae7b473c725e76a6 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/d1752cd7a73848dbae7b473c725e76a6 2024-11-14T09:16:03,183 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/d1752cd7a73848dbae7b473c725e76a6, entries=200, sequenceid=133, filesize=14.2 K 2024-11-14T09:16:03,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/3d040a1f59cc41b29d7911787c49a827 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/3d040a1f59cc41b29d7911787c49a827 2024-11-14T09:16:03,188 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/3d040a1f59cc41b29d7911787c49a827, entries=150, sequenceid=133, filesize=11.8 K 2024-11-14T09:16:03,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/71a2849408e54180989bc1873890cc3b as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/71a2849408e54180989bc1873890cc3b 2024-11-14T09:16:03,193 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/71a2849408e54180989bc1873890cc3b, entries=150, sequenceid=133, filesize=11.8 K 2024-11-14T09:16:03,194 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for add92018c22d01dad02f6ba9a2020355 in 462ms, sequenceid=133, compaction requested=true 2024-11-14T09:16:03,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for add92018c22d01dad02f6ba9a2020355: 2024-11-14T09:16:03,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store add92018c22d01dad02f6ba9a2020355:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:16:03,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:16:03,194 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:16:03,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store add92018c22d01dad02f6ba9a2020355:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:16:03,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:16:03,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store add92018c22d01dad02f6ba9a2020355:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:16:03,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:16:03,194 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:16:03,199 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38699 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:16:03,199 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:16:03,199 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): add92018c22d01dad02f6ba9a2020355/B is initiating minor compaction (all files) 2024-11-14T09:16:03,199 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): add92018c22d01dad02f6ba9a2020355/A is initiating minor compaction (all files) 2024-11-14T09:16:03,199 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of add92018c22d01dad02f6ba9a2020355/B in TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:03,199 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of add92018c22d01dad02f6ba9a2020355/A in TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:03,199 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/aae8d9e61eed42d9ba3a4476344ef224, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/f61bf08d6f4f419c9cd0eb400f1b64af, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/3d040a1f59cc41b29d7911787c49a827] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp, totalSize=35.5 K 2024-11-14T09:16:03,199 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/030bf1d59aa642fe9bedc8cf989fb5de, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/554e492687144122a63953e9af68a4cc, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/d1752cd7a73848dbae7b473c725e76a6] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp, totalSize=37.8 K 2024-11-14T09:16:03,199 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting aae8d9e61eed42d9ba3a4476344ef224, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731575757590 2024-11-14T09:16:03,199 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 030bf1d59aa642fe9bedc8cf989fb5de, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731575757590 2024-11-14T09:16:03,200 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting f61bf08d6f4f419c9cd0eb400f1b64af, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731575760983 2024-11-14T09:16:03,200 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 554e492687144122a63953e9af68a4cc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731575760983 2024-11-14T09:16:03,200 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d040a1f59cc41b29d7911787c49a827, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731575762036 2024-11-14T09:16:03,200 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1752cd7a73848dbae7b473c725e76a6, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731575762036 2024-11-14T09:16:03,208 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): add92018c22d01dad02f6ba9a2020355#B#compaction#384 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:16:03,208 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): add92018c22d01dad02f6ba9a2020355#A#compaction#385 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:16:03,208 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/1eab547bcd11421ca0c200eafc2a4ff4 is 50, key is test_row_0/B:col10/1731575762065/Put/seqid=0 2024-11-14T09:16:03,209 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/8e5aa1f28e7344a99ab8d54460a96846 is 50, key is test_row_0/A:col10/1731575762065/Put/seqid=0 2024-11-14T09:16:03,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742280_1456 (size=12409) 2024-11-14T09:16:03,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742281_1457 (size=12409) 2024-11-14T09:16:03,227 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/1eab547bcd11421ca0c200eafc2a4ff4 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/1eab547bcd11421ca0c200eafc2a4ff4 2024-11-14T09:16:03,231 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in add92018c22d01dad02f6ba9a2020355/B of add92018c22d01dad02f6ba9a2020355 into 1eab547bcd11421ca0c200eafc2a4ff4(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:16:03,231 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for add92018c22d01dad02f6ba9a2020355: 2024-11-14T09:16:03,231 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., storeName=add92018c22d01dad02f6ba9a2020355/B, priority=13, startTime=1731575763194; duration=0sec 2024-11-14T09:16:03,231 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:16:03,231 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: add92018c22d01dad02f6ba9a2020355:B 2024-11-14T09:16:03,231 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:16:03,232 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:16:03,232 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): add92018c22d01dad02f6ba9a2020355/C is initiating minor compaction (all files) 2024-11-14T09:16:03,232 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of add92018c22d01dad02f6ba9a2020355/C in TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:03,232 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/d34eabbeb03042f986c596343aa45e5b, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/928e0de62abb4c18b9a9eb412ea40a44, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/71a2849408e54180989bc1873890cc3b] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp, totalSize=35.5 K 2024-11-14T09:16:03,233 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting d34eabbeb03042f986c596343aa45e5b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731575757590 2024-11-14T09:16:03,233 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 928e0de62abb4c18b9a9eb412ea40a44, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731575760983 2024-11-14T09:16:03,233 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 71a2849408e54180989bc1873890cc3b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731575762036 2024-11-14T09:16:03,239 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): add92018c22d01dad02f6ba9a2020355#C#compaction#386 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:16:03,240 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/6f73632880ef45b5b3120a4d7c7f7a8a is 50, key is test_row_0/C:col10/1731575762065/Put/seqid=0 2024-11-14T09:16:03,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742282_1458 (size=12409) 2024-11-14T09:16:03,249 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/6f73632880ef45b5b3120a4d7c7f7a8a as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/6f73632880ef45b5b3120a4d7c7f7a8a 2024-11-14T09:16:03,255 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in add92018c22d01dad02f6ba9a2020355/C of add92018c22d01dad02f6ba9a2020355 into 6f73632880ef45b5b3120a4d7c7f7a8a(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:16:03,255 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for add92018c22d01dad02f6ba9a2020355: 2024-11-14T09:16:03,255 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., storeName=add92018c22d01dad02f6ba9a2020355/C, priority=13, startTime=1731575763194; duration=0sec 2024-11-14T09:16:03,255 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:16:03,255 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: add92018c22d01dad02f6ba9a2020355:C 2024-11-14T09:16:03,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-14T09:16:03,395 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:16:03,395 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:16:03,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-14T09:16:03,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-14T09:16:03,397 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:16:03,398 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:16:03,398 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:16:03,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on add92018c22d01dad02f6ba9a2020355 2024-11-14T09:16:03,468 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing add92018c22d01dad02f6ba9a2020355 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-14T09:16:03,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK add92018c22d01dad02f6ba9a2020355, store=A 2024-11-14T09:16:03,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:03,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK add92018c22d01dad02f6ba9a2020355, store=B 2024-11-14T09:16:03,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:03,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK add92018c22d01dad02f6ba9a2020355, store=C 2024-11-14T09:16:03,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:03,473 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/1cdbf755619a4c82aba6f7f17e8a8072 is 50, key is test_row_0/A:col10/1731575762804/Put/seqid=0 2024-11-14T09:16:03,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742283_1459 (size=14541) 2024-11-14T09:16:03,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-14T09:16:03,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-14T09:16:03,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:03,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. as already flushing 2024-11-14T09:16:03,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:03,550 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:03,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:03,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:03,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:03,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731575823576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:03,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:03,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:03,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:03,628 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/8e5aa1f28e7344a99ab8d54460a96846 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/8e5aa1f28e7344a99ab8d54460a96846 2024-11-14T09:16:03,631 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in add92018c22d01dad02f6ba9a2020355/A of add92018c22d01dad02f6ba9a2020355 into 8e5aa1f28e7344a99ab8d54460a96846(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:16:03,631 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for add92018c22d01dad02f6ba9a2020355: 2024-11-14T09:16:03,631 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., storeName=add92018c22d01dad02f6ba9a2020355/A, priority=13, startTime=1731575763194; duration=0sec 2024-11-14T09:16:03,631 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:16:03,632 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: add92018c22d01dad02f6ba9a2020355:A 2024-11-14T09:16:03,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:03,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731575823685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:03,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:03,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:03,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:03,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-14T09:16:03,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:03,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. as already flushing 2024-11-14T09:16:03,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:03,703 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:03,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:03,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:03,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-14T09:16:03,856 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-14T09:16:03,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:03,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. as already flushing 2024-11-14T09:16:03,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:03,856 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:03,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:03,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:03,878 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/1cdbf755619a4c82aba6f7f17e8a8072 2024-11-14T09:16:03,884 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/8f01fee175c34d0a944c0adcba34b4ca is 50, key is test_row_0/B:col10/1731575762804/Put/seqid=0 2024-11-14T09:16:03,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742284_1460 (size=12151) 2024-11-14T09:16:03,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:03,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731575823895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:03,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:03,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:03,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:04,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-14T09:16:04,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:04,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. as already flushing 2024-11-14T09:16:04,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:04,009 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:04,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:04,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:04,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-14T09:16:04,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-14T09:16:04,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:04,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. as already flushing 2024-11-14T09:16:04,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:04,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:04,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:04,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:04,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:04,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731575824214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:04,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:04,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:04,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:04,289 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/8f01fee175c34d0a944c0adcba34b4ca 2024-11-14T09:16:04,296 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/41ccd4309bf84ef0bef54c890113b099 is 50, key is test_row_0/C:col10/1731575762804/Put/seqid=0 2024-11-14T09:16:04,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742285_1461 (size=12151) 2024-11-14T09:16:04,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-14T09:16:04,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:04,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. as already flushing 2024-11-14T09:16:04,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:04,316 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:04,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:04,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:04,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-14T09:16:04,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:04,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. as already flushing 2024-11-14T09:16:04,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:04,469 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:04,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:04,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:04,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-14T09:16:04,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-14T09:16:04,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:04,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. as already flushing 2024-11-14T09:16:04,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:04,622 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:04,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:04,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:04,701 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/41ccd4309bf84ef0bef54c890113b099 2024-11-14T09:16:04,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/1cdbf755619a4c82aba6f7f17e8a8072 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/1cdbf755619a4c82aba6f7f17e8a8072 2024-11-14T09:16:04,708 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/1cdbf755619a4c82aba6f7f17e8a8072, entries=200, sequenceid=152, filesize=14.2 K 2024-11-14T09:16:04,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/8f01fee175c34d0a944c0adcba34b4ca as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/8f01fee175c34d0a944c0adcba34b4ca 2024-11-14T09:16:04,711 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/8f01fee175c34d0a944c0adcba34b4ca, entries=150, sequenceid=152, filesize=11.9 K 2024-11-14T09:16:04,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/41ccd4309bf84ef0bef54c890113b099 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/41ccd4309bf84ef0bef54c890113b099 2024-11-14T09:16:04,715 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/41ccd4309bf84ef0bef54c890113b099, entries=150, sequenceid=152, filesize=11.9 K 2024-11-14T09:16:04,716 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for add92018c22d01dad02f6ba9a2020355 in 1248ms, sequenceid=152, compaction requested=false 2024-11-14T09:16:04,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for add92018c22d01dad02f6ba9a2020355: 2024-11-14T09:16:04,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on add92018c22d01dad02f6ba9a2020355 2024-11-14T09:16:04,740 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing add92018c22d01dad02f6ba9a2020355 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-14T09:16:04,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK add92018c22d01dad02f6ba9a2020355, store=A 2024-11-14T09:16:04,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:04,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK add92018c22d01dad02f6ba9a2020355, store=B 2024-11-14T09:16:04,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:04,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK add92018c22d01dad02f6ba9a2020355, store=C 2024-11-14T09:16:04,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:04,745 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/4a5a04bb6f21451da8018e00df55250f is 50, key is test_row_0/A:col10/1731575764739/Put/seqid=0 2024-11-14T09:16:04,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742286_1462 (size=14541) 2024-11-14T09:16:04,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-14T09:16:04,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:04,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. as already flushing 2024-11-14T09:16:04,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:04,775 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:04,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:04,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:04,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:04,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731575824839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:04,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:04,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:04,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:04,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-14T09:16:04,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:04,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. as already flushing 2024-11-14T09:16:04,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:04,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:04,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:04,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:04,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:04,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731575824955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:04,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:04,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:04,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:05,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-14T09:16:05,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:05,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. as already flushing 2024-11-14T09:16:05,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:05,081 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:05,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:05,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:05,149 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/4a5a04bb6f21451da8018e00df55250f 2024-11-14T09:16:05,156 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/e1b16bdbe368416ab3707228ecc15f15 is 50, key is test_row_0/B:col10/1731575764739/Put/seqid=0 2024-11-14T09:16:05,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742287_1463 (size=12151) 2024-11-14T09:16:05,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:05,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731575825165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:05,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:05,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:05,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:05,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:05,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59528 deadline: 1731575825175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:05,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:05,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59508 deadline: 1731575825175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:05,183 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_1' in add92018c22d01dad02f6ba9a2020355 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:16:05,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:05,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:05,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:05,183 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_1' in add92018c22d01dad02f6ba9a2020355 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:16:05,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:05,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:05,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:05,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:05,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59578 deadline: 1731575825185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:05,188 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_2' in add92018c22d01dad02f6ba9a2020355 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:16:05,188 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:05,188 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:05,188 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:05,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:05,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1731575825194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:05,200 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_1' in add92018c22d01dad02f6ba9a2020355 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:16:05,201 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:05,201 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:05,201 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:05,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-14T09:16:05,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:05,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. as already flushing 2024-11-14T09:16:05,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:05,234 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:05,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:05,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:05,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-14T09:16:05,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:05,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. as already flushing 2024-11-14T09:16:05,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:05,387 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:05,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:05,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:05,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:05,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731575825474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:05,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:05,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:05,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:05,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-14T09:16:05,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:05,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. as already flushing 2024-11-14T09:16:05,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:05,540 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:05,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:05,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:05,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-14T09:16:05,566 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/e1b16bdbe368416ab3707228ecc15f15 2024-11-14T09:16:05,573 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/4451e6031f62472680c722d8ababaeed is 50, key is test_row_0/C:col10/1731575764739/Put/seqid=0 2024-11-14T09:16:05,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742288_1464 (size=12151) 2024-11-14T09:16:05,576 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/4451e6031f62472680c722d8ababaeed 2024-11-14T09:16:05,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/4a5a04bb6f21451da8018e00df55250f as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/4a5a04bb6f21451da8018e00df55250f 2024-11-14T09:16:05,583 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/4a5a04bb6f21451da8018e00df55250f, entries=200, sequenceid=169, filesize=14.2 K 2024-11-14T09:16:05,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/e1b16bdbe368416ab3707228ecc15f15 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/e1b16bdbe368416ab3707228ecc15f15 2024-11-14T09:16:05,586 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/e1b16bdbe368416ab3707228ecc15f15, entries=150, sequenceid=169, filesize=11.9 K 2024-11-14T09:16:05,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/4451e6031f62472680c722d8ababaeed as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/4451e6031f62472680c722d8ababaeed 2024-11-14T09:16:05,590 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/4451e6031f62472680c722d8ababaeed, entries=150, sequenceid=169, filesize=11.9 K 2024-11-14T09:16:05,591 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for add92018c22d01dad02f6ba9a2020355 in 851ms, sequenceid=169, compaction requested=true 2024-11-14T09:16:05,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for add92018c22d01dad02f6ba9a2020355: 2024-11-14T09:16:05,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store add92018c22d01dad02f6ba9a2020355:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:16:05,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:16:05,591 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:16:05,591 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:16:05,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store add92018c22d01dad02f6ba9a2020355:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:16:05,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:16:05,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store add92018c22d01dad02f6ba9a2020355:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:16:05,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:16:05,592 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41491 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:16:05,592 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:16:05,592 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): add92018c22d01dad02f6ba9a2020355/A is initiating minor compaction (all files) 2024-11-14T09:16:05,592 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): add92018c22d01dad02f6ba9a2020355/B is initiating minor compaction (all files) 2024-11-14T09:16:05,592 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of add92018c22d01dad02f6ba9a2020355/A in TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:05,592 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of add92018c22d01dad02f6ba9a2020355/B in TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:05,592 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/8e5aa1f28e7344a99ab8d54460a96846, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/1cdbf755619a4c82aba6f7f17e8a8072, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/4a5a04bb6f21451da8018e00df55250f] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp, totalSize=40.5 K 2024-11-14T09:16:05,592 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/1eab547bcd11421ca0c200eafc2a4ff4, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/8f01fee175c34d0a944c0adcba34b4ca, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/e1b16bdbe368416ab3707228ecc15f15] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp, totalSize=35.9 K 2024-11-14T09:16:05,593 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e5aa1f28e7344a99ab8d54460a96846, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731575762036 2024-11-14T09:16:05,593 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 1eab547bcd11421ca0c200eafc2a4ff4, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731575762036 2024-11-14T09:16:05,593 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f01fee175c34d0a944c0adcba34b4ca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731575762804 2024-11-14T09:16:05,593 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1cdbf755619a4c82aba6f7f17e8a8072, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731575762778 2024-11-14T09:16:05,593 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting e1b16bdbe368416ab3707228ecc15f15, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731575763537 2024-11-14T09:16:05,593 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a5a04bb6f21451da8018e00df55250f, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731575763537 2024-11-14T09:16:05,598 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): add92018c22d01dad02f6ba9a2020355#A#compaction#393 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:16:05,598 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): add92018c22d01dad02f6ba9a2020355#B#compaction#394 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:16:05,599 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/bc798c1009724027a4cd2bae520eb774 is 50, key is test_row_0/A:col10/1731575764739/Put/seqid=0 2024-11-14T09:16:05,599 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/ae0cbd80c5974df89d057fd49bc38af9 is 50, key is test_row_0/B:col10/1731575764739/Put/seqid=0 2024-11-14T09:16:05,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742289_1465 (size=12561) 2024-11-14T09:16:05,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742290_1466 (size=12561) 2024-11-14T09:16:05,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-14T09:16:05,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:05,693 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing add92018c22d01dad02f6ba9a2020355 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-14T09:16:05,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK add92018c22d01dad02f6ba9a2020355, store=A 2024-11-14T09:16:05,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:05,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK add92018c22d01dad02f6ba9a2020355, store=B 2024-11-14T09:16:05,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:05,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK add92018c22d01dad02f6ba9a2020355, store=C 2024-11-14T09:16:05,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:05,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/e3ba2b01a37a4a76afb56c2a21951a38 is 50, key is test_row_0/A:col10/1731575764774/Put/seqid=0 2024-11-14T09:16:05,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742291_1467 (size=12151) 2024-11-14T09:16:06,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on add92018c22d01dad02f6ba9a2020355 2024-11-14T09:16:06,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. as already flushing 2024-11-14T09:16:06,008 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/ae0cbd80c5974df89d057fd49bc38af9 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/ae0cbd80c5974df89d057fd49bc38af9 2024-11-14T09:16:06,009 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/bc798c1009724027a4cd2bae520eb774 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/bc798c1009724027a4cd2bae520eb774 2024-11-14T09:16:06,014 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in add92018c22d01dad02f6ba9a2020355/B of add92018c22d01dad02f6ba9a2020355 into ae0cbd80c5974df89d057fd49bc38af9(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:16:06,014 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for add92018c22d01dad02f6ba9a2020355: 2024-11-14T09:16:06,014 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., storeName=add92018c22d01dad02f6ba9a2020355/B, priority=13, startTime=1731575765591; duration=0sec 2024-11-14T09:16:06,014 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:16:06,014 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: add92018c22d01dad02f6ba9a2020355:B 2024-11-14T09:16:06,014 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:16:06,015 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in add92018c22d01dad02f6ba9a2020355/A of add92018c22d01dad02f6ba9a2020355 into bc798c1009724027a4cd2bae520eb774(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:16:06,015 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for add92018c22d01dad02f6ba9a2020355: 2024-11-14T09:16:06,015 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., storeName=add92018c22d01dad02f6ba9a2020355/A, priority=13, startTime=1731575765591; duration=0sec 2024-11-14T09:16:06,015 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:16:06,015 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: add92018c22d01dad02f6ba9a2020355:A 2024-11-14T09:16:06,016 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:16:06,016 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): add92018c22d01dad02f6ba9a2020355/C is initiating minor compaction (all files) 2024-11-14T09:16:06,016 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of add92018c22d01dad02f6ba9a2020355/C in TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:06,016 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/6f73632880ef45b5b3120a4d7c7f7a8a, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/41ccd4309bf84ef0bef54c890113b099, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/4451e6031f62472680c722d8ababaeed] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp, totalSize=35.9 K 2024-11-14T09:16:06,016 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f73632880ef45b5b3120a4d7c7f7a8a, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731575762036 2024-11-14T09:16:06,017 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 41ccd4309bf84ef0bef54c890113b099, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731575762804 2024-11-14T09:16:06,017 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 4451e6031f62472680c722d8ababaeed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731575763537 2024-11-14T09:16:06,023 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): add92018c22d01dad02f6ba9a2020355#C#compaction#396 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:16:06,024 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/02d3683de1fb4c9d9fdacbfe9af442de is 50, key is test_row_0/C:col10/1731575764739/Put/seqid=0 2024-11-14T09:16:06,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742292_1468 (size=12561) 2024-11-14T09:16:06,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:06,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731575826084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:06,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:06,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:06,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:06,101 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/e3ba2b01a37a4a76afb56c2a21951a38 2024-11-14T09:16:06,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/a2a2e49dae8f4c95b3e90c0a2f3bb22e is 50, key is test_row_0/B:col10/1731575764774/Put/seqid=0 2024-11-14T09:16:06,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742293_1469 (size=12151) 2024-11-14T09:16:06,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:06,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731575826205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:06,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:06,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:06,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:06,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:06,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731575826425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:06,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:06,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:06,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:06,438 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/02d3683de1fb4c9d9fdacbfe9af442de as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/02d3683de1fb4c9d9fdacbfe9af442de 2024-11-14T09:16:06,443 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in add92018c22d01dad02f6ba9a2020355/C of add92018c22d01dad02f6ba9a2020355 into 02d3683de1fb4c9d9fdacbfe9af442de(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:16:06,443 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for add92018c22d01dad02f6ba9a2020355: 2024-11-14T09:16:06,443 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., storeName=add92018c22d01dad02f6ba9a2020355/C, priority=13, startTime=1731575765591; duration=0sec 2024-11-14T09:16:06,443 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:16:06,443 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: add92018c22d01dad02f6ba9a2020355:C 2024-11-14T09:16:06,512 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/a2a2e49dae8f4c95b3e90c0a2f3bb22e 2024-11-14T09:16:06,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/bcec29105a7547f493289ab8d34c736f is 50, key is test_row_0/C:col10/1731575764774/Put/seqid=0 2024-11-14T09:16:06,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742294_1470 (size=12151) 2024-11-14T09:16:06,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:06,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731575826745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:06,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:06,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:06,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:06,922 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/bcec29105a7547f493289ab8d34c736f 2024-11-14T09:16:06,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/e3ba2b01a37a4a76afb56c2a21951a38 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/e3ba2b01a37a4a76afb56c2a21951a38 2024-11-14T09:16:06,929 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/e3ba2b01a37a4a76afb56c2a21951a38, entries=150, sequenceid=185, filesize=11.9 K 2024-11-14T09:16:06,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/a2a2e49dae8f4c95b3e90c0a2f3bb22e as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/a2a2e49dae8f4c95b3e90c0a2f3bb22e 2024-11-14T09:16:06,932 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/a2a2e49dae8f4c95b3e90c0a2f3bb22e, entries=150, sequenceid=185, filesize=11.9 K 2024-11-14T09:16:06,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/bcec29105a7547f493289ab8d34c736f as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/bcec29105a7547f493289ab8d34c736f 2024-11-14T09:16:06,937 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/bcec29105a7547f493289ab8d34c736f, entries=150, sequenceid=185, filesize=11.9 K 2024-11-14T09:16:06,937 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for add92018c22d01dad02f6ba9a2020355 in 1244ms, sequenceid=185, compaction requested=false 2024-11-14T09:16:06,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for add92018c22d01dad02f6ba9a2020355: 2024-11-14T09:16:06,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:06,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-14T09:16:06,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-14T09:16:06,940 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-14T09:16:06,940 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5410 sec 2024-11-14T09:16:06,942 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 3.5450 sec 2024-11-14T09:16:07,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on add92018c22d01dad02f6ba9a2020355 2024-11-14T09:16:07,268 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing add92018c22d01dad02f6ba9a2020355 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-14T09:16:07,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK add92018c22d01dad02f6ba9a2020355, store=A 2024-11-14T09:16:07,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:07,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK add92018c22d01dad02f6ba9a2020355, store=B 2024-11-14T09:16:07,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:07,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK add92018c22d01dad02f6ba9a2020355, store=C 2024-11-14T09:16:07,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:07,273 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/524580f8892643e1bb65dcba49eed355 is 50, key is test_row_0/A:col10/1731575766060/Put/seqid=0 2024-11-14T09:16:07,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742295_1471 (size=14541) 2024-11-14T09:16:07,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:07,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731575827362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:07,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:07,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:07,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:07,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:07,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731575827474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:07,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:07,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:07,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:07,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-14T09:16:07,556 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:16:07,556 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:16:07,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-14T09:16:07,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-14T09:16:07,558 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:16:07,558 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:16:07,559 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:16:07,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-14T09:16:07,677 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/524580f8892643e1bb65dcba49eed355 2024-11-14T09:16:07,684 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/6f8a9cbeece64fb2959b30017d068ce2 is 50, key is test_row_0/B:col10/1731575766060/Put/seqid=0 2024-11-14T09:16:07,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742296_1472 (size=12151) 2024-11-14T09:16:07,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:07,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731575827685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:07,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:07,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:07,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:07,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-14T09:16:07,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:07,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. as already flushing 2024-11-14T09:16:07,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:07,711 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:07,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:07,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:07,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-14T09:16:07,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:07,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. as already flushing 2024-11-14T09:16:07,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:07,864 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:07,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:07,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:07,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-14T09:16:08,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:08,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731575828004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:08,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:08,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:08,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:08,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-14T09:16:08,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:08,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. as already flushing 2024-11-14T09:16:08,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:08,017 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:08,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:08,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:08,092 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/6f8a9cbeece64fb2959b30017d068ce2 2024-11-14T09:16:08,099 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/d947a82328624ae3a2cab08a5fd7f35e is 50, key is test_row_0/C:col10/1731575766060/Put/seqid=0 2024-11-14T09:16:08,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742297_1473 (size=12151) 2024-11-14T09:16:08,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-14T09:16:08,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:08,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. as already flushing 2024-11-14T09:16:08,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:08,170 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:08,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:08,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:08,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-14T09:16:08,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-14T09:16:08,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:08,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. as already flushing 2024-11-14T09:16:08,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:08,323 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:08,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:08,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:08,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-14T09:16:08,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:08,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. as already flushing 2024-11-14T09:16:08,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:08,476 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:08,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:08,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:08,504 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/d947a82328624ae3a2cab08a5fd7f35e 2024-11-14T09:16:08,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/524580f8892643e1bb65dcba49eed355 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/524580f8892643e1bb65dcba49eed355 2024-11-14T09:16:08,511 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/524580f8892643e1bb65dcba49eed355, entries=200, sequenceid=205, filesize=14.2 K 2024-11-14T09:16:08,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/6f8a9cbeece64fb2959b30017d068ce2 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/6f8a9cbeece64fb2959b30017d068ce2 2024-11-14T09:16:08,514 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/6f8a9cbeece64fb2959b30017d068ce2, entries=150, sequenceid=205, filesize=11.9 K 2024-11-14T09:16:08,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/d947a82328624ae3a2cab08a5fd7f35e as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/d947a82328624ae3a2cab08a5fd7f35e 2024-11-14T09:16:08,518 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/d947a82328624ae3a2cab08a5fd7f35e, entries=150, sequenceid=205, filesize=11.9 K 2024-11-14T09:16:08,519 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for add92018c22d01dad02f6ba9a2020355 in 1251ms, sequenceid=205, compaction requested=true 2024-11-14T09:16:08,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for add92018c22d01dad02f6ba9a2020355: 2024-11-14T09:16:08,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store add92018c22d01dad02f6ba9a2020355:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:16:08,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:16:08,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store add92018c22d01dad02f6ba9a2020355:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:16:08,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:16:08,519 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:16:08,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store add92018c22d01dad02f6ba9a2020355:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:16:08,519 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:16:08,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:16:08,520 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:16:08,520 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39253 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:16:08,520 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): add92018c22d01dad02f6ba9a2020355/B is initiating minor compaction (all files) 2024-11-14T09:16:08,520 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): add92018c22d01dad02f6ba9a2020355/A is initiating minor compaction (all files) 2024-11-14T09:16:08,520 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of add92018c22d01dad02f6ba9a2020355/B in TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:08,520 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of add92018c22d01dad02f6ba9a2020355/A in TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:08,520 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/ae0cbd80c5974df89d057fd49bc38af9, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/a2a2e49dae8f4c95b3e90c0a2f3bb22e, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/6f8a9cbeece64fb2959b30017d068ce2] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp, totalSize=36.0 K 2024-11-14T09:16:08,520 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/bc798c1009724027a4cd2bae520eb774, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/e3ba2b01a37a4a76afb56c2a21951a38, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/524580f8892643e1bb65dcba49eed355] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp, totalSize=38.3 K 2024-11-14T09:16:08,520 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting ae0cbd80c5974df89d057fd49bc38af9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731575763537 2024-11-14T09:16:08,520 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc798c1009724027a4cd2bae520eb774, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731575763537 2024-11-14T09:16:08,521 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3ba2b01a37a4a76afb56c2a21951a38, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731575764774 2024-11-14T09:16:08,521 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting a2a2e49dae8f4c95b3e90c0a2f3bb22e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731575764774 2024-11-14T09:16:08,521 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 524580f8892643e1bb65dcba49eed355, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731575766060 2024-11-14T09:16:08,521 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f8a9cbeece64fb2959b30017d068ce2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731575766060 2024-11-14T09:16:08,528 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): add92018c22d01dad02f6ba9a2020355#A#compaction#402 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:16:08,529 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): add92018c22d01dad02f6ba9a2020355#B#compaction#403 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:16:08,529 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/a8ba38d3478a46728d3724052c8d03de is 50, key is test_row_0/A:col10/1731575766060/Put/seqid=0 2024-11-14T09:16:08,529 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/f0e2cdfefb564f80880e06da21a40386 is 50, key is test_row_0/B:col10/1731575766060/Put/seqid=0 2024-11-14T09:16:08,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on add92018c22d01dad02f6ba9a2020355 2024-11-14T09:16:08,532 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing add92018c22d01dad02f6ba9a2020355 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-14T09:16:08,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK add92018c22d01dad02f6ba9a2020355, store=A 2024-11-14T09:16:08,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:08,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK add92018c22d01dad02f6ba9a2020355, store=B 2024-11-14T09:16:08,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:08,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK add92018c22d01dad02f6ba9a2020355, store=C 2024-11-14T09:16:08,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:08,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742298_1474 (size=12663) 2024-11-14T09:16:08,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742299_1475 (size=12663) 2024-11-14T09:16:08,540 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/fa83b4efba56452cb78d58ec500dcdfc is 50, key is test_row_0/A:col10/1731575768531/Put/seqid=0 2024-11-14T09:16:08,544 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/f0e2cdfefb564f80880e06da21a40386 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/f0e2cdfefb564f80880e06da21a40386 2024-11-14T09:16:08,547 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/a8ba38d3478a46728d3724052c8d03de as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/a8ba38d3478a46728d3724052c8d03de 2024-11-14T09:16:08,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742300_1476 (size=14541) 2024-11-14T09:16:08,551 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/fa83b4efba56452cb78d58ec500dcdfc 2024-11-14T09:16:08,552 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in add92018c22d01dad02f6ba9a2020355/B of add92018c22d01dad02f6ba9a2020355 into f0e2cdfefb564f80880e06da21a40386(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:16:08,552 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for add92018c22d01dad02f6ba9a2020355: 2024-11-14T09:16:08,552 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., storeName=add92018c22d01dad02f6ba9a2020355/B, priority=13, startTime=1731575768519; duration=0sec 2024-11-14T09:16:08,552 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:16:08,552 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in add92018c22d01dad02f6ba9a2020355/A of add92018c22d01dad02f6ba9a2020355 into a8ba38d3478a46728d3724052c8d03de(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:16:08,552 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: add92018c22d01dad02f6ba9a2020355:B 2024-11-14T09:16:08,552 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for add92018c22d01dad02f6ba9a2020355: 2024-11-14T09:16:08,552 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., storeName=add92018c22d01dad02f6ba9a2020355/A, priority=13, startTime=1731575768519; duration=0sec 2024-11-14T09:16:08,552 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:16:08,552 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:16:08,552 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: add92018c22d01dad02f6ba9a2020355:A 2024-11-14T09:16:08,553 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:16:08,553 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): add92018c22d01dad02f6ba9a2020355/C is initiating minor compaction (all files) 2024-11-14T09:16:08,553 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of add92018c22d01dad02f6ba9a2020355/C in TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:08,553 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/02d3683de1fb4c9d9fdacbfe9af442de, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/bcec29105a7547f493289ab8d34c736f, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/d947a82328624ae3a2cab08a5fd7f35e] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp, totalSize=36.0 K 2024-11-14T09:16:08,553 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 02d3683de1fb4c9d9fdacbfe9af442de, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731575763537 2024-11-14T09:16:08,553 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting bcec29105a7547f493289ab8d34c736f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731575764774 2024-11-14T09:16:08,554 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting d947a82328624ae3a2cab08a5fd7f35e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731575766060 2024-11-14T09:16:08,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/480c9521df3445ec91ffb80894eabf07 is 50, key is test_row_0/B:col10/1731575768531/Put/seqid=0 2024-11-14T09:16:08,562 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): add92018c22d01dad02f6ba9a2020355#C#compaction#406 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:16:08,562 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/aa531e262a2849629bc18b0b07b57f01 is 50, key is test_row_0/C:col10/1731575766060/Put/seqid=0 2024-11-14T09:16:08,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742301_1477 (size=12151) 2024-11-14T09:16:08,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742302_1478 (size=12663) 2024-11-14T09:16:08,569 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/aa531e262a2849629bc18b0b07b57f01 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/aa531e262a2849629bc18b0b07b57f01 2024-11-14T09:16:08,573 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in add92018c22d01dad02f6ba9a2020355/C of add92018c22d01dad02f6ba9a2020355 into aa531e262a2849629bc18b0b07b57f01(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:16:08,573 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for add92018c22d01dad02f6ba9a2020355: 2024-11-14T09:16:08,573 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., storeName=add92018c22d01dad02f6ba9a2020355/C, priority=13, startTime=1731575768519; duration=0sec 2024-11-14T09:16:08,573 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:16:08,573 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: add92018c22d01dad02f6ba9a2020355:C 2024-11-14T09:16:08,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-14T09:16:08,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:08,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. as already flushing 2024-11-14T09:16:08,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:08,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:08,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:08,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:08,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:08,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731575828642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:08,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:08,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:08,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:08,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-14T09:16:08,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:08,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731575828754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:08,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:08,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:08,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:08,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-14T09:16:08,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:08,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. as already flushing 2024-11-14T09:16:08,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:08,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:08,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:08,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:08,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-14T09:16:08,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:08,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. as already flushing 2024-11-14T09:16:08,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:08,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:08,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:08,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:08,963 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/480c9521df3445ec91ffb80894eabf07 2024-11-14T09:16:08,970 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/ae3d40b3fbf94b28b13ef2e977103b0c is 50, key is test_row_0/C:col10/1731575768531/Put/seqid=0 2024-11-14T09:16:08,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:08,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731575828965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:08,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:08,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:08,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:08,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742303_1479 (size=12151) 2024-11-14T09:16:09,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-14T09:16:09,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:09,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. as already flushing 2024-11-14T09:16:09,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:09,087 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:09,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:09,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:09,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:09,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59578 deadline: 1731575829195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:09,201 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_2' in add92018c22d01dad02f6ba9a2020355 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:16:09,202 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:09,202 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:09,202 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:09,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:09,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59528 deadline: 1731575829205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:09,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:09,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59508 deadline: 1731575829205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:09,212 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_1' in add92018c22d01dad02f6ba9a2020355 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:16:09,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:09,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:09,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:09,212 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_1' in add92018c22d01dad02f6ba9a2020355 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:16:09,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:09,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:09,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:09,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:09,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1731575829214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:09,222 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_1' in add92018c22d01dad02f6ba9a2020355 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8238 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:16:09,223 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:09,223 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:09,223 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:09,240 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-14T09:16:09,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:09,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. as already flushing 2024-11-14T09:16:09,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:09,240 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:09,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:09,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:09,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:09,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731575829275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:09,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:09,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:09,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:09,375 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/ae3d40b3fbf94b28b13ef2e977103b0c 2024-11-14T09:16:09,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/fa83b4efba56452cb78d58ec500dcdfc as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/fa83b4efba56452cb78d58ec500dcdfc 2024-11-14T09:16:09,381 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/fa83b4efba56452cb78d58ec500dcdfc, entries=200, sequenceid=221, filesize=14.2 K 2024-11-14T09:16:09,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/480c9521df3445ec91ffb80894eabf07 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/480c9521df3445ec91ffb80894eabf07 2024-11-14T09:16:09,385 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/480c9521df3445ec91ffb80894eabf07, entries=150, sequenceid=221, filesize=11.9 K 2024-11-14T09:16:09,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/ae3d40b3fbf94b28b13ef2e977103b0c as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/ae3d40b3fbf94b28b13ef2e977103b0c 2024-11-14T09:16:09,389 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/ae3d40b3fbf94b28b13ef2e977103b0c, entries=150, sequenceid=221, filesize=11.9 K 2024-11-14T09:16:09,390 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for add92018c22d01dad02f6ba9a2020355 in 858ms, sequenceid=221, compaction requested=false 2024-11-14T09:16:09,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for add92018c22d01dad02f6ba9a2020355: 2024-11-14T09:16:09,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-14T09:16:09,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:09,393 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing add92018c22d01dad02f6ba9a2020355 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-14T09:16:09,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK add92018c22d01dad02f6ba9a2020355, store=A 2024-11-14T09:16:09,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:09,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK add92018c22d01dad02f6ba9a2020355, store=B 2024-11-14T09:16:09,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:09,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK add92018c22d01dad02f6ba9a2020355, store=C 2024-11-14T09:16:09,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:09,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/719862dd9d474668b423b8fb90fe5532 is 50, key is test_row_0/A:col10/1731575768634/Put/seqid=0 2024-11-14T09:16:09,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742304_1480 (size=12151) 2024-11-14T09:16:09,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-14T09:16:09,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on add92018c22d01dad02f6ba9a2020355 2024-11-14T09:16:09,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. as already flushing 2024-11-14T09:16:09,803 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/719862dd9d474668b423b8fb90fe5532 2024-11-14T09:16:09,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/9c61add143484aef8dcd965c6a006198 is 50, key is test_row_0/B:col10/1731575768634/Put/seqid=0 2024-11-14T09:16:09,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742305_1481 (size=12151) 2024-11-14T09:16:09,815 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/9c61add143484aef8dcd965c6a006198 2024-11-14T09:16:09,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/5ff13ae86d6d4927a1c67a0993373ca0 is 50, key is test_row_0/C:col10/1731575768634/Put/seqid=0 2024-11-14T09:16:09,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742306_1482 (size=12151) 2024-11-14T09:16:09,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:09,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731575829878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:09,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:09,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:09,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:10,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:10,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731575829995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:10,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:10,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:10,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:10,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:10,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731575830205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:10,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:10,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:10,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:10,225 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/5ff13ae86d6d4927a1c67a0993373ca0 2024-11-14T09:16:10,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/719862dd9d474668b423b8fb90fe5532 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/719862dd9d474668b423b8fb90fe5532 2024-11-14T09:16:10,232 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/719862dd9d474668b423b8fb90fe5532, entries=150, sequenceid=240, filesize=11.9 K 2024-11-14T09:16:10,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/9c61add143484aef8dcd965c6a006198 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/9c61add143484aef8dcd965c6a006198 2024-11-14T09:16:10,237 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/9c61add143484aef8dcd965c6a006198, entries=150, sequenceid=240, filesize=11.9 K 2024-11-14T09:16:10,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/5ff13ae86d6d4927a1c67a0993373ca0 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/5ff13ae86d6d4927a1c67a0993373ca0 2024-11-14T09:16:10,241 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/5ff13ae86d6d4927a1c67a0993373ca0, entries=150, sequenceid=240, filesize=11.9 K 2024-11-14T09:16:10,242 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for add92018c22d01dad02f6ba9a2020355 in 849ms, sequenceid=240, compaction requested=true 2024-11-14T09:16:10,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for add92018c22d01dad02f6ba9a2020355: 2024-11-14T09:16:10,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:10,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-14T09:16:10,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-14T09:16:10,245 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-14T09:16:10,245 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6850 sec 2024-11-14T09:16:10,247 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 2.6900 sec 2024-11-14T09:16:10,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on add92018c22d01dad02f6ba9a2020355 2024-11-14T09:16:10,522 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing add92018c22d01dad02f6ba9a2020355 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-14T09:16:10,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK add92018c22d01dad02f6ba9a2020355, store=A 2024-11-14T09:16:10,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:10,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK add92018c22d01dad02f6ba9a2020355, store=B 2024-11-14T09:16:10,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:10,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK add92018c22d01dad02f6ba9a2020355, store=C 2024-11-14T09:16:10,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:10,527 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/da96633b4d914380800d860e24ab2ca8 is 50, key is test_row_0/A:col10/1731575770521/Put/seqid=0 2024-11-14T09:16:10,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742307_1483 (size=14591) 2024-11-14T09:16:10,535 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/da96633b4d914380800d860e24ab2ca8 2024-11-14T09:16:10,542 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/a6741d5ed4bd4738afdb6919e58e9e61 is 50, key is test_row_0/B:col10/1731575770521/Put/seqid=0 2024-11-14T09:16:10,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742308_1484 (size=12201) 2024-11-14T09:16:10,553 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/a6741d5ed4bd4738afdb6919e58e9e61 2024-11-14T09:16:10,561 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/4a039ea2ab1e4e539709bbf24aa5b12b is 50, key is test_row_0/C:col10/1731575770521/Put/seqid=0 2024-11-14T09:16:10,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742309_1485 (size=12201) 2024-11-14T09:16:10,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:10,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731575830617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:10,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:10,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:10,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:10,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:10,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731575830735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:10,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:10,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:10,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:10,754 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T09:16:10,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:10,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731575830945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:10,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:10,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:10,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:10,965 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/4a039ea2ab1e4e539709bbf24aa5b12b 2024-11-14T09:16:10,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/da96633b4d914380800d860e24ab2ca8 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/da96633b4d914380800d860e24ab2ca8 2024-11-14T09:16:10,972 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/da96633b4d914380800d860e24ab2ca8, entries=200, sequenceid=257, filesize=14.2 K 2024-11-14T09:16:10,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/a6741d5ed4bd4738afdb6919e58e9e61 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/a6741d5ed4bd4738afdb6919e58e9e61 2024-11-14T09:16:10,975 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/a6741d5ed4bd4738afdb6919e58e9e61, entries=150, sequenceid=257, filesize=11.9 K 2024-11-14T09:16:10,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/4a039ea2ab1e4e539709bbf24aa5b12b as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/4a039ea2ab1e4e539709bbf24aa5b12b 2024-11-14T09:16:10,979 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/4a039ea2ab1e4e539709bbf24aa5b12b, entries=150, sequenceid=257, filesize=11.9 K 2024-11-14T09:16:10,980 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for add92018c22d01dad02f6ba9a2020355 in 458ms, sequenceid=257, compaction requested=true 2024-11-14T09:16:10,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for add92018c22d01dad02f6ba9a2020355: 2024-11-14T09:16:10,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store add92018c22d01dad02f6ba9a2020355:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:16:10,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:16:10,980 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:16:10,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store add92018c22d01dad02f6ba9a2020355:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:16:10,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:16:10,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store add92018c22d01dad02f6ba9a2020355:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:16:10,980 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:16:10,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:16:10,982 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49166 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:16:10,982 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:16:10,982 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): add92018c22d01dad02f6ba9a2020355/B is initiating minor compaction (all files) 2024-11-14T09:16:10,982 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): add92018c22d01dad02f6ba9a2020355/A is initiating minor compaction (all files) 2024-11-14T09:16:10,982 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of add92018c22d01dad02f6ba9a2020355/B in TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:10,982 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of add92018c22d01dad02f6ba9a2020355/A in TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:10,982 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/f0e2cdfefb564f80880e06da21a40386, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/480c9521df3445ec91ffb80894eabf07, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/9c61add143484aef8dcd965c6a006198, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/a6741d5ed4bd4738afdb6919e58e9e61] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp, totalSize=48.0 K 2024-11-14T09:16:10,982 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/a8ba38d3478a46728d3724052c8d03de, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/fa83b4efba56452cb78d58ec500dcdfc, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/719862dd9d474668b423b8fb90fe5532, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/da96633b4d914380800d860e24ab2ca8] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp, totalSize=52.7 K 2024-11-14T09:16:10,982 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting f0e2cdfefb564f80880e06da21a40386, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731575766060 2024-11-14T09:16:10,982 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8ba38d3478a46728d3724052c8d03de, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731575766060 2024-11-14T09:16:10,983 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 480c9521df3445ec91ffb80894eabf07, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731575767335 2024-11-14T09:16:10,983 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa83b4efba56452cb78d58ec500dcdfc, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731575767335 2024-11-14T09:16:10,983 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 719862dd9d474668b423b8fb90fe5532, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731575768625 2024-11-14T09:16:10,983 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c61add143484aef8dcd965c6a006198, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731575768625 2024-11-14T09:16:10,983 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting da96633b4d914380800d860e24ab2ca8, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731575769831 2024-11-14T09:16:10,983 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting a6741d5ed4bd4738afdb6919e58e9e61, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731575769831 2024-11-14T09:16:10,990 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): add92018c22d01dad02f6ba9a2020355#B#compaction#414 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:16:10,990 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/c60cc41d72314b4f9e659c1c8ec79751 is 50, key is test_row_0/B:col10/1731575770521/Put/seqid=0 2024-11-14T09:16:10,991 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): add92018c22d01dad02f6ba9a2020355#A#compaction#415 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:16:10,992 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/79a036b012384981b706c4b8e88b4722 is 50, key is test_row_0/A:col10/1731575770521/Put/seqid=0 2024-11-14T09:16:10,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742310_1486 (size=12849) 2024-11-14T09:16:10,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742311_1487 (size=12849) 2024-11-14T09:16:11,002 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/79a036b012384981b706c4b8e88b4722 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/79a036b012384981b706c4b8e88b4722 2024-11-14T09:16:11,006 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in add92018c22d01dad02f6ba9a2020355/A of add92018c22d01dad02f6ba9a2020355 into 79a036b012384981b706c4b8e88b4722(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:16:11,006 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for add92018c22d01dad02f6ba9a2020355: 2024-11-14T09:16:11,006 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., storeName=add92018c22d01dad02f6ba9a2020355/A, priority=12, startTime=1731575770980; duration=0sec 2024-11-14T09:16:11,006 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:16:11,006 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: add92018c22d01dad02f6ba9a2020355:A 2024-11-14T09:16:11,006 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:16:11,007 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49166 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:16:11,007 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): add92018c22d01dad02f6ba9a2020355/C is initiating minor compaction (all files) 2024-11-14T09:16:11,007 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of add92018c22d01dad02f6ba9a2020355/C in TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:11,007 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/aa531e262a2849629bc18b0b07b57f01, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/ae3d40b3fbf94b28b13ef2e977103b0c, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/5ff13ae86d6d4927a1c67a0993373ca0, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/4a039ea2ab1e4e539709bbf24aa5b12b] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp, totalSize=48.0 K 2024-11-14T09:16:11,008 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa531e262a2849629bc18b0b07b57f01, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731575766060 2024-11-14T09:16:11,008 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae3d40b3fbf94b28b13ef2e977103b0c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731575767335 2024-11-14T09:16:11,009 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ff13ae86d6d4927a1c67a0993373ca0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731575768625 2024-11-14T09:16:11,009 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a039ea2ab1e4e539709bbf24aa5b12b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731575769831 2024-11-14T09:16:11,017 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): add92018c22d01dad02f6ba9a2020355#C#compaction#416 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:16:11,018 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/26d8fca1e8e941339f0c6e70c14dd32c is 50, key is test_row_0/C:col10/1731575770521/Put/seqid=0 2024-11-14T09:16:11,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742312_1488 (size=12849) 2024-11-14T09:16:11,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on add92018c22d01dad02f6ba9a2020355 2024-11-14T09:16:11,271 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing add92018c22d01dad02f6ba9a2020355 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-14T09:16:11,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK add92018c22d01dad02f6ba9a2020355, store=A 2024-11-14T09:16:11,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:11,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK add92018c22d01dad02f6ba9a2020355, store=B 2024-11-14T09:16:11,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:11,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK add92018c22d01dad02f6ba9a2020355, store=C 2024-11-14T09:16:11,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:11,275 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/cb3dbf8aba414e2e8e5aa3c2136c41d0 is 50, key is test_row_0/A:col10/1731575771269/Put/seqid=0 2024-11-14T09:16:11,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742313_1489 (size=14741) 2024-11-14T09:16:11,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:11,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731575831349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:11,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:11,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:11,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:11,398 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/c60cc41d72314b4f9e659c1c8ec79751 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/c60cc41d72314b4f9e659c1c8ec79751 2024-11-14T09:16:11,402 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in add92018c22d01dad02f6ba9a2020355/B of add92018c22d01dad02f6ba9a2020355 into c60cc41d72314b4f9e659c1c8ec79751(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:16:11,402 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for add92018c22d01dad02f6ba9a2020355: 2024-11-14T09:16:11,402 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., storeName=add92018c22d01dad02f6ba9a2020355/B, priority=12, startTime=1731575770980; duration=0sec 2024-11-14T09:16:11,402 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:16:11,402 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: add92018c22d01dad02f6ba9a2020355:B 2024-11-14T09:16:11,424 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/26d8fca1e8e941339f0c6e70c14dd32c as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/26d8fca1e8e941339f0c6e70c14dd32c 2024-11-14T09:16:11,428 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in add92018c22d01dad02f6ba9a2020355/C of add92018c22d01dad02f6ba9a2020355 into 26d8fca1e8e941339f0c6e70c14dd32c(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:16:11,428 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for add92018c22d01dad02f6ba9a2020355: 2024-11-14T09:16:11,428 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., storeName=add92018c22d01dad02f6ba9a2020355/C, priority=12, startTime=1731575770980; duration=0sec 2024-11-14T09:16:11,428 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:16:11,428 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: add92018c22d01dad02f6ba9a2020355:C 2024-11-14T09:16:11,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:11,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731575831464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:11,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:11,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:11,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:11,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:11,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731575831675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:11,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:11,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:11,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:11,679 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/cb3dbf8aba414e2e8e5aa3c2136c41d0 2024-11-14T09:16:11,686 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/fb998d0b54a845f2a78ddf66d9751e49 is 50, key is test_row_0/B:col10/1731575771269/Put/seqid=0 2024-11-14T09:16:11,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742314_1490 (size=12301) 2024-11-14T09:16:11,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-14T09:16:11,715 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:16:11,715 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:16:11,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-14T09:16:11,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-14T09:16:11,717 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:16:11,718 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:16:11,718 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:16:11,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-14T09:16:11,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T09:16:11,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:11,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. as already flushing 2024-11-14T09:16:11,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:11,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:11,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:11,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:11,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:11,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731575831985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:11,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:11,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:11,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:12,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T09:16:12,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:12,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. as already flushing 2024-11-14T09:16:12,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:12,024 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:12,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:12,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:12,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-14T09:16:12,090 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/fb998d0b54a845f2a78ddf66d9751e49 2024-11-14T09:16:12,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/0119e56e3c03424a9a44bca47daf3104 is 50, key is test_row_0/C:col10/1731575771269/Put/seqid=0 2024-11-14T09:16:12,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742315_1491 (size=12301) 2024-11-14T09:16:12,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T09:16:12,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:12,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. as already flushing 2024-11-14T09:16:12,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:12,177 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:12,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:12,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:12,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T09:16:12,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:12,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. as already flushing 2024-11-14T09:16:12,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:12,329 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:12,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:12,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:12,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-14T09:16:12,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T09:16:12,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:12,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. as already flushing 2024-11-14T09:16:12,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:12,482 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:12,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:12,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:12,500 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/0119e56e3c03424a9a44bca47daf3104 2024-11-14T09:16:12,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/cb3dbf8aba414e2e8e5aa3c2136c41d0 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/cb3dbf8aba414e2e8e5aa3c2136c41d0 2024-11-14T09:16:12,507 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/cb3dbf8aba414e2e8e5aa3c2136c41d0, entries=200, sequenceid=274, filesize=14.4 K 2024-11-14T09:16:12,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/fb998d0b54a845f2a78ddf66d9751e49 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/fb998d0b54a845f2a78ddf66d9751e49 2024-11-14T09:16:12,511 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/fb998d0b54a845f2a78ddf66d9751e49, entries=150, sequenceid=274, filesize=12.0 K 2024-11-14T09:16:12,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/0119e56e3c03424a9a44bca47daf3104 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/0119e56e3c03424a9a44bca47daf3104 2024-11-14T09:16:12,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:12,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731575832505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:12,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:12,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:12,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:12,516 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/0119e56e3c03424a9a44bca47daf3104, entries=150, sequenceid=274, filesize=12.0 K 2024-11-14T09:16:12,517 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for add92018c22d01dad02f6ba9a2020355 in 1246ms, sequenceid=274, compaction requested=false 2024-11-14T09:16:12,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for add92018c22d01dad02f6ba9a2020355: 2024-11-14T09:16:12,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T09:16:12,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:12,635 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing add92018c22d01dad02f6ba9a2020355 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-14T09:16:12,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK add92018c22d01dad02f6ba9a2020355, store=A 2024-11-14T09:16:12,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:12,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK add92018c22d01dad02f6ba9a2020355, store=B 2024-11-14T09:16:12,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:12,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK add92018c22d01dad02f6ba9a2020355, store=C 2024-11-14T09:16:12,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:12,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/e56f6b864b0d440e818f003a1e5d056d is 50, key is test_row_0/A:col10/1731575771298/Put/seqid=0 2024-11-14T09:16:12,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742316_1492 (size=12301) 2024-11-14T09:16:12,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-14T09:16:13,082 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/e56f6b864b0d440e818f003a1e5d056d 2024-11-14T09:16:13,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/02d2c5cd27bb4634b936f1de6b802d64 is 50, key is test_row_0/B:col10/1731575771298/Put/seqid=0 2024-11-14T09:16:13,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742317_1493 (size=12301) 2024-11-14T09:16:13,108 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/02d2c5cd27bb4634b936f1de6b802d64 2024-11-14T09:16:13,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/22bfd499214b4f23a48098c1f85ac9c8 is 50, key is test_row_0/C:col10/1731575771298/Put/seqid=0 2024-11-14T09:16:13,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742318_1494 (size=12301) 2024-11-14T09:16:13,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on add92018c22d01dad02f6ba9a2020355 2024-11-14T09:16:13,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. as already flushing 2024-11-14T09:16:13,528 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/22bfd499214b4f23a48098c1f85ac9c8 2024-11-14T09:16:13,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/e56f6b864b0d440e818f003a1e5d056d as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/e56f6b864b0d440e818f003a1e5d056d 2024-11-14T09:16:13,536 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/e56f6b864b0d440e818f003a1e5d056d, entries=150, sequenceid=292, filesize=12.0 K 2024-11-14T09:16:13,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/02d2c5cd27bb4634b936f1de6b802d64 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/02d2c5cd27bb4634b936f1de6b802d64 2024-11-14T09:16:13,540 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/02d2c5cd27bb4634b936f1de6b802d64, entries=150, sequenceid=292, filesize=12.0 K 2024-11-14T09:16:13,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/22bfd499214b4f23a48098c1f85ac9c8 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/22bfd499214b4f23a48098c1f85ac9c8 2024-11-14T09:16:13,544 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/22bfd499214b4f23a48098c1f85ac9c8, entries=150, sequenceid=292, filesize=12.0 K 2024-11-14T09:16:13,545 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=26.84 KB/27480 for add92018c22d01dad02f6ba9a2020355 in 910ms, sequenceid=292, compaction requested=true 2024-11-14T09:16:13,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for add92018c22d01dad02f6ba9a2020355: 2024-11-14T09:16:13,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:13,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-14T09:16:13,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-14T09:16:13,549 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-14T09:16:13,549 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8290 sec 2024-11-14T09:16:13,552 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.8340 sec 2024-11-14T09:16:13,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on add92018c22d01dad02f6ba9a2020355 2024-11-14T09:16:13,560 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing add92018c22d01dad02f6ba9a2020355 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T09:16:13,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK add92018c22d01dad02f6ba9a2020355, store=A 2024-11-14T09:16:13,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:13,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK add92018c22d01dad02f6ba9a2020355, store=B 2024-11-14T09:16:13,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:13,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK add92018c22d01dad02f6ba9a2020355, store=C 2024-11-14T09:16:13,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:13,564 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/b57ce74098ff4987a141e71fcbf4b990 is 50, key is test_row_0/A:col10/1731575773538/Put/seqid=0 2024-11-14T09:16:13,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742319_1495 (size=14741) 2024-11-14T09:16:13,713 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731575773663, max timestamp=1731575773710Num uniques 859; 2024-11-14T09:16:13,713 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731575773663, max timestamp=1731575773710Num uniques 859; 2024-11-14T09:16:13,713 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:16:13,713 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:16:13,713 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731575773663, max timestamp=1731575773710Num uniques 859; 2024-11-14T09:16:13,713 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:16:13,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:13,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731575833853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:13,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:13,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:13,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:13,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-14T09:16:13,865 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:16:13,865 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:16:13,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-14T09:16:13,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-14T09:16:13,867 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:16:13,867 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:16:13,867 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:16:13,969 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/b57ce74098ff4987a141e71fcbf4b990 2024-11-14T09:16:13,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-14T09:16:13,975 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/fe2d0cb3660b4ed88bf9aefb72550287 is 50, key is test_row_0/B:col10/1731575773538/Put/seqid=0 2024-11-14T09:16:13,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:13,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731575833965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:13,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:13,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:13,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:13,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742320_1496 (size=12301) 2024-11-14T09:16:13,987 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/fe2d0cb3660b4ed88bf9aefb72550287 2024-11-14T09:16:13,995 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/09e1d87675844f90a9868d395e7932d9 is 50, key is test_row_0/C:col10/1731575773538/Put/seqid=0 2024-11-14T09:16:14,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742321_1497 (size=12301) 2024-11-14T09:16:14,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-14T09:16:14,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:14,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. as already flushing 2024-11-14T09:16:14,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:14,021 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:14,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:14,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:14,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-14T09:16:14,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:14,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. as already flushing 2024-11-14T09:16:14,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:14,174 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:14,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:14,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:14,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-14T09:16:14,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:14,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731575834185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:14,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:14,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:14,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:14,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-14T09:16:14,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:14,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. as already flushing 2024-11-14T09:16:14,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:14,331 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:14,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:14,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:14,408 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/09e1d87675844f90a9868d395e7932d9 2024-11-14T09:16:14,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/b57ce74098ff4987a141e71fcbf4b990 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/b57ce74098ff4987a141e71fcbf4b990 2024-11-14T09:16:14,418 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/b57ce74098ff4987a141e71fcbf4b990, entries=200, sequenceid=302, filesize=14.4 K 2024-11-14T09:16:14,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/fe2d0cb3660b4ed88bf9aefb72550287 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/fe2d0cb3660b4ed88bf9aefb72550287 2024-11-14T09:16:14,430 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/fe2d0cb3660b4ed88bf9aefb72550287, entries=150, sequenceid=302, filesize=12.0 K 2024-11-14T09:16:14,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/09e1d87675844f90a9868d395e7932d9 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/09e1d87675844f90a9868d395e7932d9 2024-11-14T09:16:14,433 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/09e1d87675844f90a9868d395e7932d9, entries=150, sequenceid=302, filesize=12.0 K 2024-11-14T09:16:14,434 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for add92018c22d01dad02f6ba9a2020355 in 874ms, sequenceid=302, compaction requested=true 2024-11-14T09:16:14,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for add92018c22d01dad02f6ba9a2020355: 2024-11-14T09:16:14,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store add92018c22d01dad02f6ba9a2020355:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:16:14,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:16:14,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store add92018c22d01dad02f6ba9a2020355:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:16:14,434 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:16:14,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:16:14,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store add92018c22d01dad02f6ba9a2020355:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:16:14,434 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:16:14,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:16:14,435 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54632 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:16:14,435 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): add92018c22d01dad02f6ba9a2020355/A is initiating minor compaction (all files) 2024-11-14T09:16:14,435 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of add92018c22d01dad02f6ba9a2020355/A in TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:14,436 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/79a036b012384981b706c4b8e88b4722, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/cb3dbf8aba414e2e8e5aa3c2136c41d0, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/e56f6b864b0d440e818f003a1e5d056d, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/b57ce74098ff4987a141e71fcbf4b990] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp, totalSize=53.4 K 2024-11-14T09:16:14,436 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49752 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:16:14,436 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): add92018c22d01dad02f6ba9a2020355/B is initiating minor compaction (all files) 2024-11-14T09:16:14,436 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of add92018c22d01dad02f6ba9a2020355/B in TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:14,436 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/c60cc41d72314b4f9e659c1c8ec79751, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/fb998d0b54a845f2a78ddf66d9751e49, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/02d2c5cd27bb4634b936f1de6b802d64, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/fe2d0cb3660b4ed88bf9aefb72550287] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp, totalSize=48.6 K 2024-11-14T09:16:14,436 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79a036b012384981b706c4b8e88b4722, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731575769831 2024-11-14T09:16:14,438 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting c60cc41d72314b4f9e659c1c8ec79751, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731575769831 2024-11-14T09:16:14,438 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb3dbf8aba414e2e8e5aa3c2136c41d0, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731575770584 2024-11-14T09:16:14,438 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting fb998d0b54a845f2a78ddf66d9751e49, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731575770613 2024-11-14T09:16:14,438 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting e56f6b864b0d440e818f003a1e5d056d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731575771298 2024-11-14T09:16:14,440 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 02d2c5cd27bb4634b936f1de6b802d64, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731575771298 2024-11-14T09:16:14,440 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting b57ce74098ff4987a141e71fcbf4b990, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731575773538 2024-11-14T09:16:14,440 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting fe2d0cb3660b4ed88bf9aefb72550287, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731575773538 2024-11-14T09:16:14,453 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): add92018c22d01dad02f6ba9a2020355#A#compaction#426 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:16:14,454 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/af0ea84463ba48ba804bd10a9a58b18f is 50, key is test_row_0/A:col10/1731575773538/Put/seqid=0 2024-11-14T09:16:14,459 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): add92018c22d01dad02f6ba9a2020355#B#compaction#427 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:16:14,460 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/7696741dbe1c478692cfb4f2024c30c4 is 50, key is test_row_0/B:col10/1731575773538/Put/seqid=0 2024-11-14T09:16:14,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742322_1498 (size=13085) 2024-11-14T09:16:14,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742323_1499 (size=13085) 2024-11-14T09:16:14,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-14T09:16:14,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:14,484 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing add92018c22d01dad02f6ba9a2020355 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-14T09:16:14,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK add92018c22d01dad02f6ba9a2020355, store=A 2024-11-14T09:16:14,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:16:14,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK add92018c22d01dad02f6ba9a2020355, store=B 2024-11-14T09:16:14,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:16:14,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK add92018c22d01dad02f6ba9a2020355, store=C 2024-11-14T09:16:14,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:16:14,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/740938ea19594debb08c5af296c9a2c4 is 50, key is test_row_0/A:col10/1731575773830/Put/seqid=0 2024-11-14T09:16:14,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-14T09:16:14,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742324_1500 (size=12301) 2024-11-14T09:16:14,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on add92018c22d01dad02f6ba9a2020355 2024-11-14T09:16:14,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. as already flushing 2024-11-14T09:16:14,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:14,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731575834593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:14,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:14,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:14,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:14,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:14,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731575834705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:14,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:14,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:14,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:14,875 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/af0ea84463ba48ba804bd10a9a58b18f as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/af0ea84463ba48ba804bd10a9a58b18f 2024-11-14T09:16:14,879 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in add92018c22d01dad02f6ba9a2020355/A of add92018c22d01dad02f6ba9a2020355 into af0ea84463ba48ba804bd10a9a58b18f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:16:14,879 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for add92018c22d01dad02f6ba9a2020355: 2024-11-14T09:16:14,879 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., storeName=add92018c22d01dad02f6ba9a2020355/A, priority=12, startTime=1731575774434; duration=0sec 2024-11-14T09:16:14,879 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:16:14,879 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: add92018c22d01dad02f6ba9a2020355:A 2024-11-14T09:16:14,879 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:16:14,881 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49752 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:16:14,881 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): add92018c22d01dad02f6ba9a2020355/C is initiating minor compaction (all files) 2024-11-14T09:16:14,881 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of add92018c22d01dad02f6ba9a2020355/C in TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:14,881 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/26d8fca1e8e941339f0c6e70c14dd32c, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/0119e56e3c03424a9a44bca47daf3104, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/22bfd499214b4f23a48098c1f85ac9c8, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/09e1d87675844f90a9868d395e7932d9] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp, totalSize=48.6 K 2024-11-14T09:16:14,882 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/7696741dbe1c478692cfb4f2024c30c4 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/7696741dbe1c478692cfb4f2024c30c4 2024-11-14T09:16:14,882 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26d8fca1e8e941339f0c6e70c14dd32c, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731575769831 2024-11-14T09:16:14,882 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0119e56e3c03424a9a44bca47daf3104, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731575770613 2024-11-14T09:16:14,883 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22bfd499214b4f23a48098c1f85ac9c8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731575771298 2024-11-14T09:16:14,883 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09e1d87675844f90a9868d395e7932d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731575773538 2024-11-14T09:16:14,886 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in add92018c22d01dad02f6ba9a2020355/B of add92018c22d01dad02f6ba9a2020355 into 7696741dbe1c478692cfb4f2024c30c4(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:16:14,886 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for add92018c22d01dad02f6ba9a2020355: 2024-11-14T09:16:14,886 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., storeName=add92018c22d01dad02f6ba9a2020355/B, priority=12, startTime=1731575774434; duration=0sec 2024-11-14T09:16:14,886 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:16:14,886 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: add92018c22d01dad02f6ba9a2020355:B 2024-11-14T09:16:14,895 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): add92018c22d01dad02f6ba9a2020355#C#compaction#429 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:16:14,895 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/15542c6d36e64eef95865c2c8f965c90 is 50, key is test_row_0/C:col10/1731575773538/Put/seqid=0 2024-11-14T09:16:14,898 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/740938ea19594debb08c5af296c9a2c4 2024-11-14T09:16:14,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742325_1501 (size=13085) 2024-11-14T09:16:14,909 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/15542c6d36e64eef95865c2c8f965c90 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/15542c6d36e64eef95865c2c8f965c90 2024-11-14T09:16:14,914 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in add92018c22d01dad02f6ba9a2020355/C of add92018c22d01dad02f6ba9a2020355 into 15542c6d36e64eef95865c2c8f965c90(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:16:14,914 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for add92018c22d01dad02f6ba9a2020355: 2024-11-14T09:16:14,915 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., storeName=add92018c22d01dad02f6ba9a2020355/C, priority=12, startTime=1731575774434; duration=0sec 2024-11-14T09:16:14,915 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:16:14,915 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: add92018c22d01dad02f6ba9a2020355:C 2024-11-14T09:16:14,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/85f9608f3bfc48a6bc6b02b307bc4270 is 50, key is test_row_0/B:col10/1731575773830/Put/seqid=0 2024-11-14T09:16:14,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742326_1502 (size=12301) 2024-11-14T09:16:14,936 INFO [Thread-1944 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1944. 2024-11-14T09:16:14,936 DEBUG [Thread-1944 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:16:14,936 DEBUG [Thread-1944 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:16:14,936 DEBUG [Thread-1944 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:16:14,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:14,936 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:16:14,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731575834925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:14,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:14,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:14,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:14,938 INFO [Thread-1950 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1950. 2024-11-14T09:16:14,938 DEBUG [Thread-1950 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:16:14,939 DEBUG [Thread-1950 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:16:14,939 DEBUG [Thread-1950 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:16:14,939 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:16:14,944 INFO [Thread-1946 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1946. 2024-11-14T09:16:14,945 DEBUG [Thread-1946 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:16:14,945 DEBUG [Thread-1946 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:16:14,945 DEBUG [Thread-1946 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:16:14,945 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:16:14,951 INFO [Thread-1948 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1948. 2024-11-14T09:16:14,951 DEBUG [Thread-1948 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:16:14,951 DEBUG [Thread-1948 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:16:14,951 DEBUG [Thread-1948 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:16:14,951 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:16:14,952 INFO [Thread-1952 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1952. 2024-11-14T09:16:14,952 DEBUG [Thread-1952 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:16:14,953 DEBUG [Thread-1952 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:16:14,953 DEBUG [Thread-1952 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:16:14,953 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:16:15,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-14T09:16:15,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:15,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 256 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731575835245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:15,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:15,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:15,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:15,332 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/85f9608f3bfc48a6bc6b02b307bc4270 2024-11-14T09:16:15,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/8ca7749c886242589403ec82cb589837 is 50, key is test_row_0/C:col10/1731575773830/Put/seqid=0 2024-11-14T09:16:15,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742327_1503 (size=12301) 2024-11-14T09:16:15,741 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/8ca7749c886242589403ec82cb589837 2024-11-14T09:16:15,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/740938ea19594debb08c5af296c9a2c4 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/740938ea19594debb08c5af296c9a2c4 2024-11-14T09:16:15,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:15,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 257 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731575835756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:15,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 , the old value is region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:15,757 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/740938ea19594debb08c5af296c9a2c4, entries=150, sequenceid=339, filesize=12.0 K 2024-11-14T09:16:15,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:15,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., hostname=83f56b55f2af,39927,1731575622393, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:16:15,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/85f9608f3bfc48a6bc6b02b307bc4270 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/85f9608f3bfc48a6bc6b02b307bc4270 2024-11-14T09:16:15,764 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/85f9608f3bfc48a6bc6b02b307bc4270, entries=150, sequenceid=339, filesize=12.0 K 2024-11-14T09:16:15,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/8ca7749c886242589403ec82cb589837 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/8ca7749c886242589403ec82cb589837 2024-11-14T09:16:15,771 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/8ca7749c886242589403ec82cb589837, entries=150, sequenceid=339, filesize=12.0 K 2024-11-14T09:16:15,772 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for add92018c22d01dad02f6ba9a2020355 in 1288ms, sequenceid=339, compaction requested=false 2024-11-14T09:16:15,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for add92018c22d01dad02f6ba9a2020355: 2024-11-14T09:16:15,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:15,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-14T09:16:15,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-14T09:16:15,780 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-14T09:16:15,780 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9080 sec 2024-11-14T09:16:15,782 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.9160 sec 2024-11-14T09:16:16,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-14T09:16:16,015 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:16:16,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on add92018c22d01dad02f6ba9a2020355 2024-11-14T09:16:16,776 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing add92018c22d01dad02f6ba9a2020355 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-14T09:16:16,776 INFO [Thread-1935 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1935. 2024-11-14T09:16:16,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK add92018c22d01dad02f6ba9a2020355, store=A 2024-11-14T09:16:16,776 DEBUG [Thread-1935 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:16:16,776 DEBUG [Thread-1935 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:16:16,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:16,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK add92018c22d01dad02f6ba9a2020355, store=B 2024-11-14T09:16:16,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:16,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK add92018c22d01dad02f6ba9a2020355, store=C 2024-11-14T09:16:16,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:16,776 DEBUG [Thread-1935 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:16:16,776 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:16:16,779 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/4b2d25ed2d2540ff9125913e11ff8ee5 is 50, key is test_row_0/A:col10/1731575774592/Put/seqid=0 2024-11-14T09:16:16,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742328_1504 (size=12301) 2024-11-14T09:16:17,183 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/4b2d25ed2d2540ff9125913e11ff8ee5 2024-11-14T09:16:17,188 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/34b2eebaddc84040b056bbac2f311b63 is 50, key is test_row_0/B:col10/1731575774592/Put/seqid=0 2024-11-14T09:16:17,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742329_1505 (size=12301) 2024-11-14T09:16:17,592 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/34b2eebaddc84040b056bbac2f311b63 2024-11-14T09:16:17,597 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/7f84dd66d47c4270bfb858c7a2007d0e is 50, key is test_row_0/C:col10/1731575774592/Put/seqid=0 2024-11-14T09:16:17,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742330_1506 (size=12301) 2024-11-14T09:16:18,000 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/7f84dd66d47c4270bfb858c7a2007d0e 2024-11-14T09:16:18,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/4b2d25ed2d2540ff9125913e11ff8ee5 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/4b2d25ed2d2540ff9125913e11ff8ee5 2024-11-14T09:16:18,006 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/4b2d25ed2d2540ff9125913e11ff8ee5, entries=150, sequenceid=353, filesize=12.0 K 2024-11-14T09:16:18,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/34b2eebaddc84040b056bbac2f311b63 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/34b2eebaddc84040b056bbac2f311b63 2024-11-14T09:16:18,010 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/34b2eebaddc84040b056bbac2f311b63, entries=150, sequenceid=353, filesize=12.0 K 2024-11-14T09:16:18,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/7f84dd66d47c4270bfb858c7a2007d0e as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/7f84dd66d47c4270bfb858c7a2007d0e 2024-11-14T09:16:18,013 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/7f84dd66d47c4270bfb858c7a2007d0e, entries=150, sequenceid=353, filesize=12.0 K 2024-11-14T09:16:18,014 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=0 B/0 for add92018c22d01dad02f6ba9a2020355 in 1238ms, sequenceid=353, compaction requested=true 2024-11-14T09:16:18,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for add92018c22d01dad02f6ba9a2020355: 2024-11-14T09:16:18,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store add92018c22d01dad02f6ba9a2020355:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:16:18,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:16:18,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store add92018c22d01dad02f6ba9a2020355:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:16:18,014 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:16:18,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:16:18,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store add92018c22d01dad02f6ba9a2020355:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:16:18,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:16:18,014 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:16:18,015 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:16:18,015 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:16:18,015 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): add92018c22d01dad02f6ba9a2020355/B is initiating minor compaction (all files) 2024-11-14T09:16:18,015 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): add92018c22d01dad02f6ba9a2020355/A is initiating minor compaction (all files) 2024-11-14T09:16:18,015 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of add92018c22d01dad02f6ba9a2020355/B in TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:18,015 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of add92018c22d01dad02f6ba9a2020355/A in TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:18,015 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/7696741dbe1c478692cfb4f2024c30c4, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/85f9608f3bfc48a6bc6b02b307bc4270, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/34b2eebaddc84040b056bbac2f311b63] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp, totalSize=36.8 K 2024-11-14T09:16:18,015 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/af0ea84463ba48ba804bd10a9a58b18f, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/740938ea19594debb08c5af296c9a2c4, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/4b2d25ed2d2540ff9125913e11ff8ee5] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp, totalSize=36.8 K 2024-11-14T09:16:18,015 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 7696741dbe1c478692cfb4f2024c30c4, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731575773538 2024-11-14T09:16:18,015 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting af0ea84463ba48ba804bd10a9a58b18f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731575773538 2024-11-14T09:16:18,015 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 85f9608f3bfc48a6bc6b02b307bc4270, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1731575773794 2024-11-14T09:16:18,015 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 740938ea19594debb08c5af296c9a2c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1731575773794 2024-11-14T09:16:18,015 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 34b2eebaddc84040b056bbac2f311b63, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731575774581 2024-11-14T09:16:18,015 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b2d25ed2d2540ff9125913e11ff8ee5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731575774581 2024-11-14T09:16:18,023 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): add92018c22d01dad02f6ba9a2020355#B#compaction#435 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:16:18,023 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): add92018c22d01dad02f6ba9a2020355#A#compaction#436 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:16:18,023 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/63809cf519d644449d26724dd7440cc9 is 50, key is test_row_0/A:col10/1731575774592/Put/seqid=0 2024-11-14T09:16:18,023 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/99df4f31711c4898b04343945bb5a96e is 50, key is test_row_0/B:col10/1731575774592/Put/seqid=0 2024-11-14T09:16:18,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742331_1507 (size=13187) 2024-11-14T09:16:18,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742332_1508 (size=13187) 2024-11-14T09:16:18,430 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/99df4f31711c4898b04343945bb5a96e as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/99df4f31711c4898b04343945bb5a96e 2024-11-14T09:16:18,430 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/63809cf519d644449d26724dd7440cc9 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/63809cf519d644449d26724dd7440cc9 2024-11-14T09:16:18,434 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in add92018c22d01dad02f6ba9a2020355/B of add92018c22d01dad02f6ba9a2020355 into 99df4f31711c4898b04343945bb5a96e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:16:18,434 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in add92018c22d01dad02f6ba9a2020355/A of add92018c22d01dad02f6ba9a2020355 into 63809cf519d644449d26724dd7440cc9(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:16:18,434 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for add92018c22d01dad02f6ba9a2020355: 2024-11-14T09:16:18,434 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for add92018c22d01dad02f6ba9a2020355: 2024-11-14T09:16:18,434 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., storeName=add92018c22d01dad02f6ba9a2020355/B, priority=13, startTime=1731575778014; duration=0sec 2024-11-14T09:16:18,434 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., storeName=add92018c22d01dad02f6ba9a2020355/A, priority=13, startTime=1731575778014; duration=0sec 2024-11-14T09:16:18,434 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:16:18,434 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: add92018c22d01dad02f6ba9a2020355:B 2024-11-14T09:16:18,434 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:16:18,434 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: add92018c22d01dad02f6ba9a2020355:A 2024-11-14T09:16:18,434 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:16:18,435 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:16:18,435 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): add92018c22d01dad02f6ba9a2020355/C is initiating minor compaction (all files) 2024-11-14T09:16:18,435 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of add92018c22d01dad02f6ba9a2020355/C in TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:18,435 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/15542c6d36e64eef95865c2c8f965c90, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/8ca7749c886242589403ec82cb589837, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/7f84dd66d47c4270bfb858c7a2007d0e] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp, totalSize=36.8 K 2024-11-14T09:16:18,435 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 15542c6d36e64eef95865c2c8f965c90, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731575773538 2024-11-14T09:16:18,435 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ca7749c886242589403ec82cb589837, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1731575773794 2024-11-14T09:16:18,436 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f84dd66d47c4270bfb858c7a2007d0e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731575774581 2024-11-14T09:16:18,441 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): add92018c22d01dad02f6ba9a2020355#C#compaction#437 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:16:18,441 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/4c4c4a2e895743b2aff84bb63e1e0f7f is 50, key is test_row_0/C:col10/1731575774592/Put/seqid=0 2024-11-14T09:16:18,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742333_1509 (size=13187) 2024-11-14T09:16:18,848 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/4c4c4a2e895743b2aff84bb63e1e0f7f as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/4c4c4a2e895743b2aff84bb63e1e0f7f 2024-11-14T09:16:18,851 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in add92018c22d01dad02f6ba9a2020355/C of add92018c22d01dad02f6ba9a2020355 into 4c4c4a2e895743b2aff84bb63e1e0f7f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:16:18,851 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for add92018c22d01dad02f6ba9a2020355: 2024-11-14T09:16:18,852 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., storeName=add92018c22d01dad02f6ba9a2020355/C, priority=13, startTime=1731575778014; duration=0sec 2024-11-14T09:16:18,852 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:16:18,852 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: add92018c22d01dad02f6ba9a2020355:C 2024-11-14T09:16:19,237 INFO [Thread-1937 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1937. 2024-11-14T09:16:19,237 DEBUG [Thread-1937 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:16:19,237 DEBUG [Thread-1937 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:16:19,237 DEBUG [Thread-1937 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:16:19,237 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:16:19,256 INFO [Thread-1941 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1941. 2024-11-14T09:16:19,256 DEBUG [Thread-1941 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:16:19,256 DEBUG [Thread-1941 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:16:19,256 DEBUG [Thread-1941 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:16:19,256 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:16:19,276 INFO [Thread-1933 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1933. 2024-11-14T09:16:19,276 DEBUG [Thread-1933 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:16:19,276 DEBUG [Thread-1933 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:16:19,276 DEBUG [Thread-1933 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:16:19,276 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:16:19,286 INFO [Thread-1939 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1939. 2024-11-14T09:16:19,286 DEBUG [Thread-1939 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:16:19,286 DEBUG [Thread-1939 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:16:19,286 DEBUG [Thread-1939 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:16:19,286 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:16:19,286 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T09:16:19,286 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 22 2024-11-14T09:16:19,286 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 196 2024-11-14T09:16:19,286 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 23 2024-11-14T09:16:19,286 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-11-14T09:16:19,286 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 20 2024-11-14T09:16:19,286 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T09:16:19,286 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T09:16:19,286 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1189 2024-11-14T09:16:19,286 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3563 rows 2024-11-14T09:16:19,286 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1191 2024-11-14T09:16:19,286 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3573 rows 2024-11-14T09:16:19,286 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1187 2024-11-14T09:16:19,286 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3561 rows 2024-11-14T09:16:19,286 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1191 2024-11-14T09:16:19,286 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3570 rows 2024-11-14T09:16:19,287 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1192 2024-11-14T09:16:19,287 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3575 rows 2024-11-14T09:16:19,287 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T09:16:19,287 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T09:16:19,287 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:16:19,287 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:16:19,287 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:16:19,287 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-14T09:16:19,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T09:16:19,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-14T09:16:19,290 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731575779290"}]},"ts":"1731575779290"} 2024-11-14T09:16:19,291 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T09:16:19,291 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T09:16:19,291 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T09:16:19,292 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=add92018c22d01dad02f6ba9a2020355, UNASSIGN}] 2024-11-14T09:16:19,293 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=add92018c22d01dad02f6ba9a2020355, UNASSIGN 2024-11-14T09:16:19,294 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=add92018c22d01dad02f6ba9a2020355, regionState=CLOSING, regionLocation=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:19,295 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=add92018c22d01dad02f6ba9a2020355, UNASSIGN because future has completed 2024-11-14T09:16:19,295 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T09:16:19,295 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; CloseRegionProcedure add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393}] 2024-11-14T09:16:19,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-14T09:16:19,447 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(122): Close add92018c22d01dad02f6ba9a2020355 2024-11-14T09:16:19,447 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T09:16:19,447 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1722): Closing add92018c22d01dad02f6ba9a2020355, disabling compactions & flushes 2024-11-14T09:16:19,447 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:19,447 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:19,447 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. after waiting 0 ms 2024-11-14T09:16:19,447 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:19,447 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(2902): Flushing add92018c22d01dad02f6ba9a2020355 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-14T09:16:19,448 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK add92018c22d01dad02f6ba9a2020355, store=A 2024-11-14T09:16:19,448 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:19,448 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK add92018c22d01dad02f6ba9a2020355, store=B 2024-11-14T09:16:19,448 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:19,448 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK add92018c22d01dad02f6ba9a2020355, store=C 2024-11-14T09:16:19,448 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:19,451 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/f554b45d4613479db0c8962d13f16034 is 50, key is test_row_1/A:col10/1731575779285/Put/seqid=0 2024-11-14T09:16:19,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742334_1510 (size=9857) 2024-11-14T09:16:19,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-14T09:16:19,855 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/f554b45d4613479db0c8962d13f16034 2024-11-14T09:16:19,860 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/738fe753abb14e77ae004858329a10ef is 50, key is test_row_1/B:col10/1731575779285/Put/seqid=0 2024-11-14T09:16:19,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742335_1511 (size=9857) 2024-11-14T09:16:19,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-14T09:16:20,264 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/738fe753abb14e77ae004858329a10ef 2024-11-14T09:16:20,269 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/a815181171884dde85fcd33ad695eeba is 50, key is test_row_1/C:col10/1731575779285/Put/seqid=0 2024-11-14T09:16:20,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742336_1512 (size=9857) 2024-11-14T09:16:20,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-14T09:16:20,675 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/a815181171884dde85fcd33ad695eeba 2024-11-14T09:16:20,678 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/A/f554b45d4613479db0c8962d13f16034 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/f554b45d4613479db0c8962d13f16034 2024-11-14T09:16:20,680 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/f554b45d4613479db0c8962d13f16034, entries=100, sequenceid=363, filesize=9.6 K 2024-11-14T09:16:20,681 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/B/738fe753abb14e77ae004858329a10ef as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/738fe753abb14e77ae004858329a10ef 2024-11-14T09:16:20,683 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/738fe753abb14e77ae004858329a10ef, entries=100, sequenceid=363, filesize=9.6 K 2024-11-14T09:16:20,683 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/.tmp/C/a815181171884dde85fcd33ad695eeba as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/a815181171884dde85fcd33ad695eeba 2024-11-14T09:16:20,686 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/a815181171884dde85fcd33ad695eeba, entries=100, sequenceid=363, filesize=9.6 K 2024-11-14T09:16:20,686 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for add92018c22d01dad02f6ba9a2020355 in 1239ms, sequenceid=363, compaction requested=false 2024-11-14T09:16:20,687 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/ba37d9ac90774ee29895aaada4a4e3fa, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/26bf1c95ed1b49e8b45b39fc20248a18, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/02135b1ac4e74ee48e3b9d0d841131c6, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/debed659e8f54619937a415a7d17d7e4, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/2188ece040b347a49101b9b2dfa4f70a, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/030bf1d59aa642fe9bedc8cf989fb5de, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/1001a22b2ff04723bd6f6f18e0e12002, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/554e492687144122a63953e9af68a4cc, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/d1752cd7a73848dbae7b473c725e76a6, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/8e5aa1f28e7344a99ab8d54460a96846, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/1cdbf755619a4c82aba6f7f17e8a8072, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/4a5a04bb6f21451da8018e00df55250f, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/bc798c1009724027a4cd2bae520eb774, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/e3ba2b01a37a4a76afb56c2a21951a38, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/524580f8892643e1bb65dcba49eed355, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/a8ba38d3478a46728d3724052c8d03de, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/fa83b4efba56452cb78d58ec500dcdfc, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/719862dd9d474668b423b8fb90fe5532, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/da96633b4d914380800d860e24ab2ca8, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/79a036b012384981b706c4b8e88b4722, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/cb3dbf8aba414e2e8e5aa3c2136c41d0, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/e56f6b864b0d440e818f003a1e5d056d, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/b57ce74098ff4987a141e71fcbf4b990, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/af0ea84463ba48ba804bd10a9a58b18f, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/740938ea19594debb08c5af296c9a2c4, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/4b2d25ed2d2540ff9125913e11ff8ee5] to archive 2024-11-14T09:16:20,687 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T09:16:20,689 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/ba37d9ac90774ee29895aaada4a4e3fa to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/ba37d9ac90774ee29895aaada4a4e3fa 2024-11-14T09:16:20,689 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/26bf1c95ed1b49e8b45b39fc20248a18 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/26bf1c95ed1b49e8b45b39fc20248a18 2024-11-14T09:16:20,690 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/02135b1ac4e74ee48e3b9d0d841131c6 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/02135b1ac4e74ee48e3b9d0d841131c6 2024-11-14T09:16:20,691 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/debed659e8f54619937a415a7d17d7e4 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/debed659e8f54619937a415a7d17d7e4 2024-11-14T09:16:20,691 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/2188ece040b347a49101b9b2dfa4f70a to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/2188ece040b347a49101b9b2dfa4f70a 2024-11-14T09:16:20,692 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/030bf1d59aa642fe9bedc8cf989fb5de to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/030bf1d59aa642fe9bedc8cf989fb5de 2024-11-14T09:16:20,693 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/1001a22b2ff04723bd6f6f18e0e12002 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/1001a22b2ff04723bd6f6f18e0e12002 2024-11-14T09:16:20,694 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/554e492687144122a63953e9af68a4cc to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/554e492687144122a63953e9af68a4cc 2024-11-14T09:16:20,694 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/d1752cd7a73848dbae7b473c725e76a6 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/d1752cd7a73848dbae7b473c725e76a6 2024-11-14T09:16:20,695 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/8e5aa1f28e7344a99ab8d54460a96846 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/8e5aa1f28e7344a99ab8d54460a96846 2024-11-14T09:16:20,696 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/1cdbf755619a4c82aba6f7f17e8a8072 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/1cdbf755619a4c82aba6f7f17e8a8072 2024-11-14T09:16:20,696 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/4a5a04bb6f21451da8018e00df55250f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/4a5a04bb6f21451da8018e00df55250f 2024-11-14T09:16:20,697 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/bc798c1009724027a4cd2bae520eb774 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/bc798c1009724027a4cd2bae520eb774 2024-11-14T09:16:20,698 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/e3ba2b01a37a4a76afb56c2a21951a38 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/e3ba2b01a37a4a76afb56c2a21951a38 2024-11-14T09:16:20,698 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/524580f8892643e1bb65dcba49eed355 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/524580f8892643e1bb65dcba49eed355 2024-11-14T09:16:20,699 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/a8ba38d3478a46728d3724052c8d03de to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/a8ba38d3478a46728d3724052c8d03de 2024-11-14T09:16:20,700 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/fa83b4efba56452cb78d58ec500dcdfc to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/fa83b4efba56452cb78d58ec500dcdfc 2024-11-14T09:16:20,700 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/719862dd9d474668b423b8fb90fe5532 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/719862dd9d474668b423b8fb90fe5532 2024-11-14T09:16:20,701 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/da96633b4d914380800d860e24ab2ca8 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/da96633b4d914380800d860e24ab2ca8 2024-11-14T09:16:20,702 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/79a036b012384981b706c4b8e88b4722 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/79a036b012384981b706c4b8e88b4722 2024-11-14T09:16:20,703 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/cb3dbf8aba414e2e8e5aa3c2136c41d0 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/cb3dbf8aba414e2e8e5aa3c2136c41d0 2024-11-14T09:16:20,703 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/e56f6b864b0d440e818f003a1e5d056d to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/e56f6b864b0d440e818f003a1e5d056d 2024-11-14T09:16:20,704 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/b57ce74098ff4987a141e71fcbf4b990 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/b57ce74098ff4987a141e71fcbf4b990 2024-11-14T09:16:20,705 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/af0ea84463ba48ba804bd10a9a58b18f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/af0ea84463ba48ba804bd10a9a58b18f 2024-11-14T09:16:20,706 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/740938ea19594debb08c5af296c9a2c4 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/740938ea19594debb08c5af296c9a2c4 2024-11-14T09:16:20,706 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/4b2d25ed2d2540ff9125913e11ff8ee5 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/4b2d25ed2d2540ff9125913e11ff8ee5 2024-11-14T09:16:20,708 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/d817f3936ba1404f88da0f57fbdb9818, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/b7b74dc119ef473db227fc4219651f94, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/990b7d3155354ea1bfa11fd705c525aa, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/3be4aa6b98bb412fb38eaa8c92a0b02f, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/d7e6c10f2cbe445392c0a4b5e08eef7e, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/aae8d9e61eed42d9ba3a4476344ef224, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/055b51dd4ab24453a712599caf9e8e06, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/f61bf08d6f4f419c9cd0eb400f1b64af, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/1eab547bcd11421ca0c200eafc2a4ff4, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/3d040a1f59cc41b29d7911787c49a827, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/8f01fee175c34d0a944c0adcba34b4ca, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/ae0cbd80c5974df89d057fd49bc38af9, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/e1b16bdbe368416ab3707228ecc15f15, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/a2a2e49dae8f4c95b3e90c0a2f3bb22e, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/f0e2cdfefb564f80880e06da21a40386, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/6f8a9cbeece64fb2959b30017d068ce2, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/480c9521df3445ec91ffb80894eabf07, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/9c61add143484aef8dcd965c6a006198, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/c60cc41d72314b4f9e659c1c8ec79751, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/a6741d5ed4bd4738afdb6919e58e9e61, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/fb998d0b54a845f2a78ddf66d9751e49, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/02d2c5cd27bb4634b936f1de6b802d64, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/7696741dbe1c478692cfb4f2024c30c4, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/fe2d0cb3660b4ed88bf9aefb72550287, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/85f9608f3bfc48a6bc6b02b307bc4270, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/34b2eebaddc84040b056bbac2f311b63] to archive 2024-11-14T09:16:20,708 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T09:16:20,709 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/d817f3936ba1404f88da0f57fbdb9818 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/d817f3936ba1404f88da0f57fbdb9818 2024-11-14T09:16:20,710 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/b7b74dc119ef473db227fc4219651f94 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/b7b74dc119ef473db227fc4219651f94 2024-11-14T09:16:20,711 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/990b7d3155354ea1bfa11fd705c525aa to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/990b7d3155354ea1bfa11fd705c525aa 2024-11-14T09:16:20,712 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/3be4aa6b98bb412fb38eaa8c92a0b02f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/3be4aa6b98bb412fb38eaa8c92a0b02f 2024-11-14T09:16:20,712 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/d7e6c10f2cbe445392c0a4b5e08eef7e to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/d7e6c10f2cbe445392c0a4b5e08eef7e 2024-11-14T09:16:20,713 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/aae8d9e61eed42d9ba3a4476344ef224 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/aae8d9e61eed42d9ba3a4476344ef224 2024-11-14T09:16:20,714 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/055b51dd4ab24453a712599caf9e8e06 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/055b51dd4ab24453a712599caf9e8e06 2024-11-14T09:16:20,715 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/f61bf08d6f4f419c9cd0eb400f1b64af to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/f61bf08d6f4f419c9cd0eb400f1b64af 2024-11-14T09:16:20,715 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/1eab547bcd11421ca0c200eafc2a4ff4 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/1eab547bcd11421ca0c200eafc2a4ff4 2024-11-14T09:16:20,716 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/3d040a1f59cc41b29d7911787c49a827 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/3d040a1f59cc41b29d7911787c49a827 2024-11-14T09:16:20,717 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/8f01fee175c34d0a944c0adcba34b4ca to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/8f01fee175c34d0a944c0adcba34b4ca 2024-11-14T09:16:20,718 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/ae0cbd80c5974df89d057fd49bc38af9 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/ae0cbd80c5974df89d057fd49bc38af9 2024-11-14T09:16:20,719 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/e1b16bdbe368416ab3707228ecc15f15 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/e1b16bdbe368416ab3707228ecc15f15 2024-11-14T09:16:20,719 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/a2a2e49dae8f4c95b3e90c0a2f3bb22e to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/a2a2e49dae8f4c95b3e90c0a2f3bb22e 2024-11-14T09:16:20,720 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/f0e2cdfefb564f80880e06da21a40386 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/f0e2cdfefb564f80880e06da21a40386 2024-11-14T09:16:20,721 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/6f8a9cbeece64fb2959b30017d068ce2 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/6f8a9cbeece64fb2959b30017d068ce2 2024-11-14T09:16:20,722 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/480c9521df3445ec91ffb80894eabf07 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/480c9521df3445ec91ffb80894eabf07 2024-11-14T09:16:20,722 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/9c61add143484aef8dcd965c6a006198 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/9c61add143484aef8dcd965c6a006198 2024-11-14T09:16:20,723 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/c60cc41d72314b4f9e659c1c8ec79751 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/c60cc41d72314b4f9e659c1c8ec79751 2024-11-14T09:16:20,724 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/a6741d5ed4bd4738afdb6919e58e9e61 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/a6741d5ed4bd4738afdb6919e58e9e61 2024-11-14T09:16:20,725 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/fb998d0b54a845f2a78ddf66d9751e49 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/fb998d0b54a845f2a78ddf66d9751e49 2024-11-14T09:16:20,725 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/02d2c5cd27bb4634b936f1de6b802d64 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/02d2c5cd27bb4634b936f1de6b802d64 2024-11-14T09:16:20,726 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/7696741dbe1c478692cfb4f2024c30c4 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/7696741dbe1c478692cfb4f2024c30c4 2024-11-14T09:16:20,727 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/fe2d0cb3660b4ed88bf9aefb72550287 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/fe2d0cb3660b4ed88bf9aefb72550287 2024-11-14T09:16:20,728 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/85f9608f3bfc48a6bc6b02b307bc4270 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/85f9608f3bfc48a6bc6b02b307bc4270 2024-11-14T09:16:20,728 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/34b2eebaddc84040b056bbac2f311b63 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/34b2eebaddc84040b056bbac2f311b63 2024-11-14T09:16:20,729 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/ee992efded7442eda59e0967421678df, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/8e9825e6700d4756ac9bceb4b1766bc2, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/90a828f277c24dfcbe91b7dc2171743b, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/30707c155e5a4e33bf486a51a99167f0, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/27cb1bd010f24e04a0a674d60208953c, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/d34eabbeb03042f986c596343aa45e5b, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/b5c24bad483d43bfabbb5e283f17c7c3, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/928e0de62abb4c18b9a9eb412ea40a44, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/6f73632880ef45b5b3120a4d7c7f7a8a, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/71a2849408e54180989bc1873890cc3b, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/41ccd4309bf84ef0bef54c890113b099, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/02d3683de1fb4c9d9fdacbfe9af442de, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/4451e6031f62472680c722d8ababaeed, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/bcec29105a7547f493289ab8d34c736f, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/aa531e262a2849629bc18b0b07b57f01, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/d947a82328624ae3a2cab08a5fd7f35e, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/ae3d40b3fbf94b28b13ef2e977103b0c, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/5ff13ae86d6d4927a1c67a0993373ca0, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/26d8fca1e8e941339f0c6e70c14dd32c, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/4a039ea2ab1e4e539709bbf24aa5b12b, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/0119e56e3c03424a9a44bca47daf3104, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/22bfd499214b4f23a48098c1f85ac9c8, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/15542c6d36e64eef95865c2c8f965c90, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/09e1d87675844f90a9868d395e7932d9, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/8ca7749c886242589403ec82cb589837, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/7f84dd66d47c4270bfb858c7a2007d0e] to archive 2024-11-14T09:16:20,730 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T09:16:20,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/ee992efded7442eda59e0967421678df to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/ee992efded7442eda59e0967421678df 2024-11-14T09:16:20,732 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/8e9825e6700d4756ac9bceb4b1766bc2 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/8e9825e6700d4756ac9bceb4b1766bc2 2024-11-14T09:16:20,733 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/90a828f277c24dfcbe91b7dc2171743b to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/90a828f277c24dfcbe91b7dc2171743b 2024-11-14T09:16:20,733 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/30707c155e5a4e33bf486a51a99167f0 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/30707c155e5a4e33bf486a51a99167f0 2024-11-14T09:16:20,734 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/27cb1bd010f24e04a0a674d60208953c to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/27cb1bd010f24e04a0a674d60208953c 2024-11-14T09:16:20,735 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/d34eabbeb03042f986c596343aa45e5b to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/d34eabbeb03042f986c596343aa45e5b 2024-11-14T09:16:20,736 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/b5c24bad483d43bfabbb5e283f17c7c3 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/b5c24bad483d43bfabbb5e283f17c7c3 2024-11-14T09:16:20,736 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/928e0de62abb4c18b9a9eb412ea40a44 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/928e0de62abb4c18b9a9eb412ea40a44 2024-11-14T09:16:20,737 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/6f73632880ef45b5b3120a4d7c7f7a8a to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/6f73632880ef45b5b3120a4d7c7f7a8a 2024-11-14T09:16:20,738 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/71a2849408e54180989bc1873890cc3b to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/71a2849408e54180989bc1873890cc3b 2024-11-14T09:16:20,739 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/41ccd4309bf84ef0bef54c890113b099 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/41ccd4309bf84ef0bef54c890113b099 2024-11-14T09:16:20,740 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/02d3683de1fb4c9d9fdacbfe9af442de to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/02d3683de1fb4c9d9fdacbfe9af442de 2024-11-14T09:16:20,741 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/4451e6031f62472680c722d8ababaeed to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/4451e6031f62472680c722d8ababaeed 2024-11-14T09:16:20,741 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/bcec29105a7547f493289ab8d34c736f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/bcec29105a7547f493289ab8d34c736f 2024-11-14T09:16:20,742 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/aa531e262a2849629bc18b0b07b57f01 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/aa531e262a2849629bc18b0b07b57f01 2024-11-14T09:16:20,743 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/d947a82328624ae3a2cab08a5fd7f35e to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/d947a82328624ae3a2cab08a5fd7f35e 2024-11-14T09:16:20,744 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/ae3d40b3fbf94b28b13ef2e977103b0c to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/ae3d40b3fbf94b28b13ef2e977103b0c 2024-11-14T09:16:20,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/5ff13ae86d6d4927a1c67a0993373ca0 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/5ff13ae86d6d4927a1c67a0993373ca0 2024-11-14T09:16:20,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/26d8fca1e8e941339f0c6e70c14dd32c to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/26d8fca1e8e941339f0c6e70c14dd32c 2024-11-14T09:16:20,746 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/4a039ea2ab1e4e539709bbf24aa5b12b to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/4a039ea2ab1e4e539709bbf24aa5b12b 2024-11-14T09:16:20,747 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/0119e56e3c03424a9a44bca47daf3104 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/0119e56e3c03424a9a44bca47daf3104 2024-11-14T09:16:20,747 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/22bfd499214b4f23a48098c1f85ac9c8 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/22bfd499214b4f23a48098c1f85ac9c8 2024-11-14T09:16:20,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/15542c6d36e64eef95865c2c8f965c90 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/15542c6d36e64eef95865c2c8f965c90 2024-11-14T09:16:20,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/09e1d87675844f90a9868d395e7932d9 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/09e1d87675844f90a9868d395e7932d9 2024-11-14T09:16:20,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/8ca7749c886242589403ec82cb589837 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/8ca7749c886242589403ec82cb589837 2024-11-14T09:16:20,751 DEBUG [StoreCloser-TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/7f84dd66d47c4270bfb858c7a2007d0e to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/7f84dd66d47c4270bfb858c7a2007d0e 2024-11-14T09:16:20,754 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/recovered.edits/366.seqid, newMaxSeqId=366, maxSeqId=1 2024-11-14T09:16:20,755 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. 2024-11-14T09:16:20,755 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1676): Region close journal for add92018c22d01dad02f6ba9a2020355: Waiting for close lock at 1731575779447Running coprocessor pre-close hooks at 1731575779447Disabling compacts and flushes for region at 1731575779447Disabling writes for close at 1731575779447Obtaining lock to block concurrent updates at 1731575779447Preparing flush snapshotting stores in add92018c22d01dad02f6ba9a2020355 at 1731575779447Finished memstore snapshotting TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731575779448 (+1 ms)Flushing stores of TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355. at 1731575779448Flushing add92018c22d01dad02f6ba9a2020355/A: creating writer at 1731575779448Flushing add92018c22d01dad02f6ba9a2020355/A: appending metadata at 1731575779450 (+2 ms)Flushing add92018c22d01dad02f6ba9a2020355/A: closing flushed file at 1731575779450Flushing add92018c22d01dad02f6ba9a2020355/B: creating writer at 1731575779858 (+408 ms)Flushing add92018c22d01dad02f6ba9a2020355/B: appending metadata at 1731575779860 (+2 ms)Flushing add92018c22d01dad02f6ba9a2020355/B: closing flushed file at 1731575779860Flushing add92018c22d01dad02f6ba9a2020355/C: creating writer at 1731575780267 (+407 ms)Flushing add92018c22d01dad02f6ba9a2020355/C: appending metadata at 1731575780268 (+1 ms)Flushing add92018c22d01dad02f6ba9a2020355/C: closing flushed file at 1731575780269 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7996512: reopening flushed file at 1731575780678 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6e43e7eb: reopening flushed file at 1731575780680 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@da75d9d: reopening flushed file at 1731575780683 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for add92018c22d01dad02f6ba9a2020355 in 1239ms, sequenceid=363, compaction requested=false at 1731575780686 (+3 ms)Writing region close event to WAL at 1731575780752 (+66 ms)Running coprocessor post-close hooks at 1731575780755 (+3 ms)Closed at 1731575780755 2024-11-14T09:16:20,757 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(157): Closed add92018c22d01dad02f6ba9a2020355 2024-11-14T09:16:20,757 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=add92018c22d01dad02f6ba9a2020355, regionState=CLOSED 2024-11-14T09:16:20,758 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=134, ppid=133, state=RUNNABLE, hasLock=false; CloseRegionProcedure add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 because future has completed 2024-11-14T09:16:20,760 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-14T09:16:20,760 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; CloseRegionProcedure add92018c22d01dad02f6ba9a2020355, server=83f56b55f2af,39927,1731575622393 in 1.4640 sec 2024-11-14T09:16:20,761 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=133, resume processing ppid=132 2024-11-14T09:16:20,762 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, ppid=132, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=add92018c22d01dad02f6ba9a2020355, UNASSIGN in 1.4680 sec 2024-11-14T09:16:20,763 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-14T09:16:20,763 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4710 sec 2024-11-14T09:16:20,764 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731575780764"}]},"ts":"1731575780764"} 2024-11-14T09:16:20,765 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T09:16:20,765 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T09:16:20,767 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4790 sec 2024-11-14T09:16:21,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-14T09:16:21,435 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:16:21,435 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-14T09:16:21,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:16:21,437 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=135, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:16:21,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-14T09:16:21,438 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=135, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:16:21,440 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355 2024-11-14T09:16:21,441 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A, FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B, FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C, FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/recovered.edits] 2024-11-14T09:16:21,443 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/63809cf519d644449d26724dd7440cc9 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/63809cf519d644449d26724dd7440cc9 2024-11-14T09:16:21,444 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/f554b45d4613479db0c8962d13f16034 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/A/f554b45d4613479db0c8962d13f16034 2024-11-14T09:16:21,446 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/738fe753abb14e77ae004858329a10ef to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/738fe753abb14e77ae004858329a10ef 2024-11-14T09:16:21,447 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/99df4f31711c4898b04343945bb5a96e to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/B/99df4f31711c4898b04343945bb5a96e 2024-11-14T09:16:21,449 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/4c4c4a2e895743b2aff84bb63e1e0f7f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/4c4c4a2e895743b2aff84bb63e1e0f7f 2024-11-14T09:16:21,450 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/a815181171884dde85fcd33ad695eeba to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/C/a815181171884dde85fcd33ad695eeba 2024-11-14T09:16:21,452 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/recovered.edits/366.seqid to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355/recovered.edits/366.seqid 2024-11-14T09:16:21,452 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/add92018c22d01dad02f6ba9a2020355 2024-11-14T09:16:21,452 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T09:16:21,454 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=135, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:16:21,456 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T09:16:21,457 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T09:16:21,458 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=135, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:16:21,458 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T09:16:21,458 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731575781458"}]},"ts":"9223372036854775807"} 2024-11-14T09:16:21,460 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T09:16:21,460 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => add92018c22d01dad02f6ba9a2020355, NAME => 'TestAcidGuarantees,,1731575752743.add92018c22d01dad02f6ba9a2020355.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T09:16:21,460 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T09:16:21,460 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731575781460"}]},"ts":"9223372036854775807"} 2024-11-14T09:16:21,461 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T09:16:21,461 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=135, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:16:21,462 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 27 msec 2024-11-14T09:16:21,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-14T09:16:21,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T09:16:21,546 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:16:21,555 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250 (was 249) - Thread LEAK? -, OpenFileDescriptor=455 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=415 (was 455), ProcessCount=11 (was 11), AvailableMemoryMB=6181 (was 6234) 2024-11-14T09:16:21,563 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=250, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=415, ProcessCount=11, AvailableMemoryMB=6181 2024-11-14T09:16:21,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T09:16:21,564 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T09:16:21,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=136, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T09:16:21,565 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T09:16:21,566 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:21,566 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 136 2024-11-14T09:16:21,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-14T09:16:21,566 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T09:16:21,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742337_1513 (size=1061) 2024-11-14T09:16:21,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-14T09:16:21,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-14T09:16:21,973 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db 2024-11-14T09:16:21,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742338_1514 (size=53) 2024-11-14T09:16:22,114 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-14T09:16:22,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-14T09:16:22,378 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:16:22,378 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 1d18b03686124f021810b831ea32e069, disabling compactions & flushes 2024-11-14T09:16:22,378 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:22,378 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:22,378 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. after waiting 0 ms 2024-11-14T09:16:22,378 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:22,378 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:22,378 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1d18b03686124f021810b831ea32e069: Waiting for close lock at 1731575782378Disabling compacts and flushes for region at 1731575782378Disabling writes for close at 1731575782378Writing region close event to WAL at 1731575782378Closed at 1731575782378 2024-11-14T09:16:22,379 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T09:16:22,379 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731575782379"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731575782379"}]},"ts":"1731575782379"} 2024-11-14T09:16:22,381 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T09:16:22,381 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T09:16:22,381 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731575782381"}]},"ts":"1731575782381"} 2024-11-14T09:16:22,382 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T09:16:22,383 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d18b03686124f021810b831ea32e069, ASSIGN}] 2024-11-14T09:16:22,383 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d18b03686124f021810b831ea32e069, ASSIGN 2024-11-14T09:16:22,384 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d18b03686124f021810b831ea32e069, ASSIGN; state=OFFLINE, location=83f56b55f2af,39927,1731575622393; forceNewPlan=false, retain=false 2024-11-14T09:16:22,535 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=1d18b03686124f021810b831ea32e069, regionState=OPENING, regionLocation=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:22,536 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d18b03686124f021810b831ea32e069, ASSIGN because future has completed 2024-11-14T09:16:22,537 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393}] 2024-11-14T09:16:22,691 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:22,691 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(7752): Opening region: {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} 2024-11-14T09:16:22,691 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1d18b03686124f021810b831ea32e069 2024-11-14T09:16:22,691 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:16:22,691 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(7794): checking encryption for 1d18b03686124f021810b831ea32e069 2024-11-14T09:16:22,691 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(7797): checking classloading for 1d18b03686124f021810b831ea32e069 2024-11-14T09:16:22,692 INFO [StoreOpener-1d18b03686124f021810b831ea32e069-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1d18b03686124f021810b831ea32e069 2024-11-14T09:16:22,693 INFO [StoreOpener-1d18b03686124f021810b831ea32e069-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T09:16:22,694 INFO [StoreOpener-1d18b03686124f021810b831ea32e069-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1d18b03686124f021810b831ea32e069 columnFamilyName A 2024-11-14T09:16:22,694 DEBUG [StoreOpener-1d18b03686124f021810b831ea32e069-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:22,694 INFO [StoreOpener-1d18b03686124f021810b831ea32e069-1 {}] regionserver.HStore(327): Store=1d18b03686124f021810b831ea32e069/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:16:22,694 INFO [StoreOpener-1d18b03686124f021810b831ea32e069-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1d18b03686124f021810b831ea32e069 2024-11-14T09:16:22,695 INFO [StoreOpener-1d18b03686124f021810b831ea32e069-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T09:16:22,695 INFO [StoreOpener-1d18b03686124f021810b831ea32e069-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1d18b03686124f021810b831ea32e069 columnFamilyName B 2024-11-14T09:16:22,695 DEBUG [StoreOpener-1d18b03686124f021810b831ea32e069-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:22,695 INFO [StoreOpener-1d18b03686124f021810b831ea32e069-1 {}] regionserver.HStore(327): Store=1d18b03686124f021810b831ea32e069/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:16:22,695 INFO [StoreOpener-1d18b03686124f021810b831ea32e069-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1d18b03686124f021810b831ea32e069 2024-11-14T09:16:22,696 INFO [StoreOpener-1d18b03686124f021810b831ea32e069-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T09:16:22,696 INFO [StoreOpener-1d18b03686124f021810b831ea32e069-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1d18b03686124f021810b831ea32e069 columnFamilyName C 2024-11-14T09:16:22,696 DEBUG [StoreOpener-1d18b03686124f021810b831ea32e069-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:22,696 INFO [StoreOpener-1d18b03686124f021810b831ea32e069-1 {}] regionserver.HStore(327): Store=1d18b03686124f021810b831ea32e069/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:16:22,697 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:22,697 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1038): replaying wal for 1d18b03686124f021810b831ea32e069 2024-11-14T09:16:22,697 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069 2024-11-14T09:16:22,697 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069 2024-11-14T09:16:22,698 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1048): stopping wal replay for 1d18b03686124f021810b831ea32e069 2024-11-14T09:16:22,698 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1060): Cleaning up temporary data for 1d18b03686124f021810b831ea32e069 2024-11-14T09:16:22,698 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T09:16:22,699 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1093): writing seq id for 1d18b03686124f021810b831ea32e069 2024-11-14T09:16:22,700 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T09:16:22,701 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1114): Opened 1d18b03686124f021810b831ea32e069; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73596491, jitterRate=0.09667317569255829}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T09:16:22,701 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1d18b03686124f021810b831ea32e069 2024-11-14T09:16:22,701 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1006): Region open journal for 1d18b03686124f021810b831ea32e069: Running coprocessor pre-open hook at 1731575782691Writing region info on filesystem at 1731575782691Initializing all the Stores at 1731575782692 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731575782692Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731575782692Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731575782692Cleaning up temporary data from old regions at 1731575782698 (+6 ms)Running coprocessor post-open hooks at 1731575782701 (+3 ms)Region opened successfully at 1731575782701 2024-11-14T09:16:22,702 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., pid=138, masterSystemTime=1731575782688 2024-11-14T09:16:22,703 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:22,703 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:22,704 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=1d18b03686124f021810b831ea32e069, regionState=OPEN, openSeqNum=2, regionLocation=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:22,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-14T09:16:22,705 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=138, ppid=137, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 because future has completed 2024-11-14T09:16:22,707 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-14T09:16:22,707 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; OpenRegionProcedure 1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 in 169 msec 2024-11-14T09:16:22,708 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=136 2024-11-14T09:16:22,709 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=136, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d18b03686124f021810b831ea32e069, ASSIGN in 325 msec 2024-11-14T09:16:22,709 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T09:16:22,709 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731575782709"}]},"ts":"1731575782709"} 2024-11-14T09:16:22,710 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T09:16:22,711 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T09:16:22,712 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1470 sec 2024-11-14T09:16:23,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-14T09:16:23,715 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:16:23,716 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@185b2823, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:16:23,716 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:16:23,716 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:16:23,717 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:16:23,717 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:16:23,717 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:16:23,717 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f5e672c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:16:23,717 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:16:23,718 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:16:23,718 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:16:23,718 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60348, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:16:23,719 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2be68894, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:16:23,719 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:16:23,720 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:16:23,720 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:16:23,721 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58080, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:16:23,721 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T09:16:23,722 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 83f56b55f2af,45325,1731575621660 2024-11-14T09:16:23,722 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7e90dd30 2024-11-14T09:16:23,722 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T09:16:23,723 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60360, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T09:16:23,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T09:16:23,724 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T09:16:23,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-14T09:16:23,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742339_1515 (size=1097) 2024-11-14T09:16:24,134 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-14T09:16:24,134 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-14T09:16:24,136 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T09:16:24,138 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d18b03686124f021810b831ea32e069, REOPEN/MOVE}] 2024-11-14T09:16:24,139 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d18b03686124f021810b831ea32e069, REOPEN/MOVE 2024-11-14T09:16:24,139 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=1d18b03686124f021810b831ea32e069, regionState=CLOSING, regionLocation=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:24,140 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d18b03686124f021810b831ea32e069, REOPEN/MOVE because future has completed 2024-11-14T09:16:24,141 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T09:16:24,141 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393}] 2024-11-14T09:16:24,293 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(122): Close 1d18b03686124f021810b831ea32e069 2024-11-14T09:16:24,293 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T09:16:24,293 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1722): Closing 1d18b03686124f021810b831ea32e069, disabling compactions & flushes 2024-11-14T09:16:24,293 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:24,293 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:24,293 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. after waiting 0 ms 2024-11-14T09:16:24,293 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:24,296 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T09:16:24,296 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:24,296 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1676): Region close journal for 1d18b03686124f021810b831ea32e069: Waiting for close lock at 1731575784293Running coprocessor pre-close hooks at 1731575784293Disabling compacts and flushes for region at 1731575784293Disabling writes for close at 1731575784293Writing region close event to WAL at 1731575784293Running coprocessor post-close hooks at 1731575784296 (+3 ms)Closed at 1731575784296 2024-11-14T09:16:24,296 WARN [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionServer(3299): Not adding moved region record: 1d18b03686124f021810b831ea32e069 to self. 2024-11-14T09:16:24,298 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(157): Closed 1d18b03686124f021810b831ea32e069 2024-11-14T09:16:24,298 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=1d18b03686124f021810b831ea32e069, regionState=CLOSED 2024-11-14T09:16:24,299 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 because future has completed 2024-11-14T09:16:24,301 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-14T09:16:24,301 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseRegionProcedure 1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 in 159 msec 2024-11-14T09:16:24,302 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d18b03686124f021810b831ea32e069, REOPEN/MOVE; state=CLOSED, location=83f56b55f2af,39927,1731575622393; forceNewPlan=false, retain=true 2024-11-14T09:16:24,452 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=1d18b03686124f021810b831ea32e069, regionState=OPENING, regionLocation=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:24,454 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d18b03686124f021810b831ea32e069, REOPEN/MOVE because future has completed 2024-11-14T09:16:24,454 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393}] 2024-11-14T09:16:24,608 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:24,608 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(7752): Opening region: {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} 2024-11-14T09:16:24,608 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1d18b03686124f021810b831ea32e069 2024-11-14T09:16:24,608 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:16:24,608 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(7794): checking encryption for 1d18b03686124f021810b831ea32e069 2024-11-14T09:16:24,608 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(7797): checking classloading for 1d18b03686124f021810b831ea32e069 2024-11-14T09:16:24,609 INFO [StoreOpener-1d18b03686124f021810b831ea32e069-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1d18b03686124f021810b831ea32e069 2024-11-14T09:16:24,610 INFO [StoreOpener-1d18b03686124f021810b831ea32e069-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T09:16:24,610 INFO [StoreOpener-1d18b03686124f021810b831ea32e069-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1d18b03686124f021810b831ea32e069 columnFamilyName A 2024-11-14T09:16:24,611 DEBUG [StoreOpener-1d18b03686124f021810b831ea32e069-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:24,611 INFO [StoreOpener-1d18b03686124f021810b831ea32e069-1 {}] regionserver.HStore(327): Store=1d18b03686124f021810b831ea32e069/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:16:24,611 INFO [StoreOpener-1d18b03686124f021810b831ea32e069-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1d18b03686124f021810b831ea32e069 2024-11-14T09:16:24,612 INFO [StoreOpener-1d18b03686124f021810b831ea32e069-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T09:16:24,612 INFO [StoreOpener-1d18b03686124f021810b831ea32e069-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1d18b03686124f021810b831ea32e069 columnFamilyName B 2024-11-14T09:16:24,612 DEBUG [StoreOpener-1d18b03686124f021810b831ea32e069-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:24,612 INFO [StoreOpener-1d18b03686124f021810b831ea32e069-1 {}] regionserver.HStore(327): Store=1d18b03686124f021810b831ea32e069/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:16:24,612 INFO [StoreOpener-1d18b03686124f021810b831ea32e069-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1d18b03686124f021810b831ea32e069 2024-11-14T09:16:24,613 INFO [StoreOpener-1d18b03686124f021810b831ea32e069-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T09:16:24,613 INFO [StoreOpener-1d18b03686124f021810b831ea32e069-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1d18b03686124f021810b831ea32e069 columnFamilyName C 2024-11-14T09:16:24,613 DEBUG [StoreOpener-1d18b03686124f021810b831ea32e069-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:24,613 INFO [StoreOpener-1d18b03686124f021810b831ea32e069-1 {}] regionserver.HStore(327): Store=1d18b03686124f021810b831ea32e069/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:16:24,613 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:24,613 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1038): replaying wal for 1d18b03686124f021810b831ea32e069 2024-11-14T09:16:24,614 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069 2024-11-14T09:16:24,614 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069 2024-11-14T09:16:24,615 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1048): stopping wal replay for 1d18b03686124f021810b831ea32e069 2024-11-14T09:16:24,615 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1060): Cleaning up temporary data for 1d18b03686124f021810b831ea32e069 2024-11-14T09:16:24,616 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T09:16:24,617 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1093): writing seq id for 1d18b03686124f021810b831ea32e069 2024-11-14T09:16:24,617 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1114): Opened 1d18b03686124f021810b831ea32e069; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61209624, jitterRate=-0.08790552616119385}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T09:16:24,617 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1d18b03686124f021810b831ea32e069 2024-11-14T09:16:24,622 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1006): Region open journal for 1d18b03686124f021810b831ea32e069: Running coprocessor pre-open hook at 1731575784608Writing region info on filesystem at 1731575784608Initializing all the Stores at 1731575784609 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731575784609Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731575784609Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731575784609Cleaning up temporary data from old regions at 1731575784615 (+6 ms)Running coprocessor post-open hooks at 1731575784617 (+2 ms)Region opened successfully at 1731575784622 (+5 ms) 2024-11-14T09:16:24,623 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., pid=143, masterSystemTime=1731575784605 2024-11-14T09:16:24,624 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:24,624 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:24,625 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=1d18b03686124f021810b831ea32e069, regionState=OPEN, openSeqNum=5, regionLocation=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:24,626 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 because future has completed 2024-11-14T09:16:24,627 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=141 2024-11-14T09:16:24,627 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=141, state=SUCCESS, hasLock=false; OpenRegionProcedure 1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 in 172 msec 2024-11-14T09:16:24,629 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-14T09:16:24,629 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d18b03686124f021810b831ea32e069, REOPEN/MOVE in 489 msec 2024-11-14T09:16:24,631 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-14T09:16:24,631 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 493 msec 2024-11-14T09:16:24,632 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 907 msec 2024-11-14T09:16:24,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-14T09:16:24,633 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:16:24,634 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d29c9e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:16:24,634 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:16:24,634 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:16:24,635 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:16:24,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:16:24,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:16:24,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18f6dfac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:16:24,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:16:24,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:16:24,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:16:24,636 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60380, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:16:24,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ebebe29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:16:24,637 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1572f078, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:16:24,637 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:16:24,637 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:16:24,638 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:16:24,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:16:24,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:16:24,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@339fad42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:16:24,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:16:24,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:16:24,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:16:24,639 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60390, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:16:24,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51e19c61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:16:24,640 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23448b6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:16:24,640 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:16:24,640 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:16:24,641 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:16:24,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:16:24,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:16:24,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@336172fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:16:24,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:16:24,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:16:24,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:16:24,642 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60418, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:16:24,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@aece4bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:16:24,643 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d7db06e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:16:24,643 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:16:24,643 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:16:24,643 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:16:24,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:16:24,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:16:24,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@661e10d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:16:24,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:16:24,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:16:24,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:16:24,644 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60444, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:16:24,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28a9cfa6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:16:24,645 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@763ca8cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:16:24,645 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:16:24,645 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:16:24,646 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:16:24,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:16:24,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:16:24,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78d12955, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:16:24,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:16:24,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:16:24,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:16:24,647 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60454, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:16:24,647 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b827efd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:16:24,648 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5451f0cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:16:24,648 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:16:24,648 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:16:24,648 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:16:24,648 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:16:24,649 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:16:24,649 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ea9dabd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:16:24,649 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:16:24,649 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:16:24,649 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:16:24,649 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60476, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:16:24,650 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@648d9fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:16:24,650 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b0e0fb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:16:24,650 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:16:24,650 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:16:24,651 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:16:24,651 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:16:24,651 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:16:24,651 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37208a6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:16:24,651 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:16:24,651 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:16:24,652 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:16:24,652 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60508, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:16:24,652 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79cb59da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:16:24,653 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cbb8a41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:16:24,653 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:16:24,653 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:16:24,654 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:16:24,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:16:24,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:16:24,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c1ba88b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:16:24,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:16:24,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:16:24,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:16:24,655 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60516, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:16:24,655 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6eefd29f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:16:24,655 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31a58f58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:16:24,656 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:16:24,656 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:16:24,656 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:16:24,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:16:24,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:16:24,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f4e2b00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:16:24,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:16:24,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:16:24,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:16:24,657 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60530, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:16:24,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e0359ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:16:24,658 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e766e1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:16:24,658 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,45325,-1 for getting cluster id 2024-11-14T09:16:24,658 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:16:24,659 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13edf1ec-b933-4893-841a-d61c78de83f6' 2024-11-14T09:16:24,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:16:24,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13edf1ec-b933-4893-841a-d61c78de83f6" 2024-11-14T09:16:24,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e4ded8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:16:24,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,45325,-1] 2024-11-14T09:16:24,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:16:24,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:16:24,660 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60550, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:16:24,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b51f9b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:16:24,661 DEBUG [Thread-2327 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:16:24,661 DEBUG [Thread-2329 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:16:24,661 DEBUG [Thread-2320 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:16:24,661 DEBUG [Thread-2318 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:16:24,661 DEBUG [Thread-2314 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:16:24,661 DEBUG [Thread-2333 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:16:24,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:16:24,662 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:16:24,662 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:16:24,662 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:16:24,662 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:16:24,662 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:16:24,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:16:24,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:16:24,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:16:24,662 DEBUG [Thread-2316 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:16:24,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:16:24,662 DEBUG [Thread-2322 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:16:24,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:16:24,662 DEBUG [Thread-2325 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:16:24,663 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58082, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:16:24,663 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:16:24,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:16:24,663 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:16:24,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:16:24,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:16:24,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:16:24,664 DEBUG [Thread-2331 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:16:24,664 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58090, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:16:24,664 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58098, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:16:24,664 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58126, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:16:24,664 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58136, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:16:24,664 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:16:24,665 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:16:24,665 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58142, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:16:24,665 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58114, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:16:24,665 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58156, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:16:24,665 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58140, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:16:24,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,39927,1731575622393, seqNum=-1] 2024-11-14T09:16:24,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:16:24,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5] 2024-11-14T09:16:24,666 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5] 2024-11-14T09:16:24,666 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58162, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:16:24,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees 2024-11-14T09:16:24,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5] 2024-11-14T09:16:24,667 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5] 2024-11-14T09:16:24,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5] 2024-11-14T09:16:24,668 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5] 2024-11-14T09:16:24,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5] 2024-11-14T09:16:24,669 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5] 2024-11-14T09:16:24,669 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:16:24,670 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:16:24,670 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:16:24,670 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5] 2024-11-14T09:16:24,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-14T09:16:24,672 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5] 2024-11-14T09:16:24,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on 1d18b03686124f021810b831ea32e069 2024-11-14T09:16:24,676 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1d18b03686124f021810b831ea32e069 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T09:16:24,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d18b03686124f021810b831ea32e069, store=A 2024-11-14T09:16:24,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:24,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d18b03686124f021810b831ea32e069, store=B 2024-11-14T09:16:24,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:24,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d18b03686124f021810b831ea32e069, store=C 2024-11-14T09:16:24,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:24,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:24,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1731575784685, max timestamp=1731575784698Num uniques 859; 2024-11-14T09:16:24,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T09:16:24,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731575844697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:24,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1731575784685, max timestamp=1731575784698Num uniques 859; 2024-11-14T09:16:24,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1731575784685, max timestamp=1731575784698Num uniques 859; 2024-11-14T09:16:24,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T09:16:24,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T09:16:24,702 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:24,703 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:24,703 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:24,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114281785b254da421abdaf2905e5855ed3_1d18b03686124f021810b831ea32e069 is 50, key is test_row_0/A:col10/1731575784670/Put/seqid=0 2024-11-14T09:16:24,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:24,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731575844722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:24,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:24,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:24,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:24,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:24,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58162 deadline: 1731575844724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:24,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:24,728 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:24,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58142 deadline: 1731575844724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:24,728 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:24,728 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:24,729 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:24,729 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:24,729 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:24,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742340_1516 (size=12154) 2024-11-14T09:16:24,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:24,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1731575844727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:24,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:24,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:24,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:24,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-14T09:16:24,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:24,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731575844805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:24,807 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:24,807 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:24,807 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:24,824 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-14T09:16:24,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:24,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. as already flushing 2024-11-14T09:16:24,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:24,824 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:24,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:24,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:24,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:24,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731575844834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:24,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:24,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:24,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:24,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:24,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58162 deadline: 1731575844834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:24,837 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:24,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:24,837 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:24,837 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:24,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58142 deadline: 1731575844834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:24,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:24,838 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:24,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1731575844835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:24,838 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:24,838 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:24,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:24,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:24,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:24,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-14T09:16:24,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:24,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. as already flushing 2024-11-14T09:16:24,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:24,977 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:24,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:24,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:24,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-14T09:16:25,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:25,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731575845015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:25,018 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:25,019 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:25,019 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:25,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:25,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731575845044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:25,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:25,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:25,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58162 deadline: 1731575845045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:25,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:25,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:25,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:25,048 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:25,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58142 deadline: 1731575845045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:25,048 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:25,049 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:25,049 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:25,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:25,049 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:25,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1731575845045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:25,049 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:25,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:25,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:25,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:25,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-14T09:16:25,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:25,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. as already flushing 2024-11-14T09:16:25,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:25,130 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:25,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:25,130 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:25,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:25,134 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114281785b254da421abdaf2905e5855ed3_1d18b03686124f021810b831ea32e069 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114281785b254da421abdaf2905e5855ed3_1d18b03686124f021810b831ea32e069 2024-11-14T09:16:25,135 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/cdb74091cb0c43338a5bbaa1a49955d0, store: [table=TestAcidGuarantees family=A region=1d18b03686124f021810b831ea32e069] 2024-11-14T09:16:25,136 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/cdb74091cb0c43338a5bbaa1a49955d0 is 175, key is test_row_0/A:col10/1731575784670/Put/seqid=0 2024-11-14T09:16:25,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742341_1517 (size=30955) 2024-11-14T09:16:25,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-14T09:16:25,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:25,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. as already flushing 2024-11-14T09:16:25,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:25,283 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:25,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:25,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:25,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-14T09:16:25,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:25,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731575845325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:25,329 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:25,329 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:25,329 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:25,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:25,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731575845354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:25,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:25,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:25,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58162 deadline: 1731575845354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:25,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:25,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:25,359 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:25,359 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:25,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:25,359 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:25,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58142 deadline: 1731575845354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:25,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:25,359 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:25,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1731575845355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:25,359 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:25,359 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:25,359 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:25,359 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:25,359 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:25,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-14T09:16:25,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:25,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. as already flushing 2024-11-14T09:16:25,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:25,436 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:25,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:25,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:25,539 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/cdb74091cb0c43338a5bbaa1a49955d0 2024-11-14T09:16:25,560 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/abb0b3279ea14c4797b2b2a3c1951ff7 is 50, key is test_row_0/B:col10/1731575784670/Put/seqid=0 2024-11-14T09:16:25,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742342_1518 (size=12001) 2024-11-14T09:16:25,573 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/abb0b3279ea14c4797b2b2a3c1951ff7 2024-11-14T09:16:25,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-14T09:16:25,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:25,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. as already flushing 2024-11-14T09:16:25,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:25,589 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:25,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:25,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:25,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/4f3bf9b5135e4f20b04059ed9aa8c7fa is 50, key is test_row_0/C:col10/1731575784670/Put/seqid=0 2024-11-14T09:16:25,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742343_1519 (size=12001) 2024-11-14T09:16:25,740 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-14T09:16:25,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:25,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. as already flushing 2024-11-14T09:16:25,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:25,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:25,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:25,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:25,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-14T09:16:25,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:25,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731575845834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:25,837 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:25,837 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:25,838 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:25,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:25,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731575845865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:25,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:25,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58162 deadline: 1731575845865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:25,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:25,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:25,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:25,869 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:25,869 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:25,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:25,869 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:25,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58142 deadline: 1731575845865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:25,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:25,869 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:25,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1731575845865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:25,869 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:25,869 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:25,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:25,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:25,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:25,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-14T09:16:25,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:25,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. as already flushing 2024-11-14T09:16:25,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:25,893 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:25,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:25,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:25,999 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/4f3bf9b5135e4f20b04059ed9aa8c7fa 2024-11-14T09:16:26,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/cdb74091cb0c43338a5bbaa1a49955d0 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/cdb74091cb0c43338a5bbaa1a49955d0 2024-11-14T09:16:26,006 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/cdb74091cb0c43338a5bbaa1a49955d0, entries=150, sequenceid=14, filesize=30.2 K 2024-11-14T09:16:26,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/abb0b3279ea14c4797b2b2a3c1951ff7 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/abb0b3279ea14c4797b2b2a3c1951ff7 2024-11-14T09:16:26,009 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/abb0b3279ea14c4797b2b2a3c1951ff7, entries=150, sequenceid=14, filesize=11.7 K 2024-11-14T09:16:26,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/4f3bf9b5135e4f20b04059ed9aa8c7fa as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/4f3bf9b5135e4f20b04059ed9aa8c7fa 2024-11-14T09:16:26,013 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/4f3bf9b5135e4f20b04059ed9aa8c7fa, entries=150, sequenceid=14, filesize=11.7 K 2024-11-14T09:16:26,014 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 1d18b03686124f021810b831ea32e069 in 1338ms, sequenceid=14, compaction requested=false 2024-11-14T09:16:26,014 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-14T09:16:26,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1d18b03686124f021810b831ea32e069: 2024-11-14T09:16:26,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-14T09:16:26,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:26,046 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2902): Flushing 1d18b03686124f021810b831ea32e069 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-14T09:16:26,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d18b03686124f021810b831ea32e069, store=A 2024-11-14T09:16:26,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:16:26,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d18b03686124f021810b831ea32e069, store=B 2024-11-14T09:16:26,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:16:26,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d18b03686124f021810b831ea32e069, store=C 2024-11-14T09:16:26,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:16:26,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111444cdbab773f446d3be787a9b068ae885_1d18b03686124f021810b831ea32e069 is 50, key is test_row_0/A:col10/1731575784722/Put/seqid=0 2024-11-14T09:16:26,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742344_1520 (size=12154) 2024-11-14T09:16:26,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,061 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111444cdbab773f446d3be787a9b068ae885_1d18b03686124f021810b831ea32e069 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111444cdbab773f446d3be787a9b068ae885_1d18b03686124f021810b831ea32e069 2024-11-14T09:16:26,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/924d28422b2443b3a0ecc9735ba1c183, store: [table=TestAcidGuarantees family=A region=1d18b03686124f021810b831ea32e069] 2024-11-14T09:16:26,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/924d28422b2443b3a0ecc9735ba1c183 is 175, key is test_row_0/A:col10/1731575784722/Put/seqid=0 2024-11-14T09:16:26,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742345_1521 (size=30955) 2024-11-14T09:16:26,274 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T09:16:26,468 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/924d28422b2443b3a0ecc9735ba1c183 2024-11-14T09:16:26,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/b1277aac80824eceab1c93889bc13d1d is 50, key is test_row_0/B:col10/1731575784722/Put/seqid=0 2024-11-14T09:16:26,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742346_1522 (size=12001) 2024-11-14T09:16:26,495 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/b1277aac80824eceab1c93889bc13d1d 2024-11-14T09:16:26,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/8d41bb25d20245718e2e0075b2c38301 is 50, key is test_row_0/C:col10/1731575784722/Put/seqid=0 2024-11-14T09:16:26,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742347_1523 (size=12001) 2024-11-14T09:16:26,524 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/8d41bb25d20245718e2e0075b2c38301 2024-11-14T09:16:26,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/924d28422b2443b3a0ecc9735ba1c183 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/924d28422b2443b3a0ecc9735ba1c183 2024-11-14T09:16:26,541 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/924d28422b2443b3a0ecc9735ba1c183, entries=150, sequenceid=49, filesize=30.2 K 2024-11-14T09:16:26,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/b1277aac80824eceab1c93889bc13d1d as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/b1277aac80824eceab1c93889bc13d1d 2024-11-14T09:16:26,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,547 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/b1277aac80824eceab1c93889bc13d1d, entries=150, sequenceid=49, filesize=11.7 K 2024-11-14T09:16:26,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/8d41bb25d20245718e2e0075b2c38301 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/8d41bb25d20245718e2e0075b2c38301 2024-11-14T09:16:26,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,560 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/8d41bb25d20245718e2e0075b2c38301, entries=150, sequenceid=49, filesize=11.7 K 2024-11-14T09:16:26,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,561 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=0 B/0 for 1d18b03686124f021810b831ea32e069 in 514ms, sequenceid=49, compaction requested=false 2024-11-14T09:16:26,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2603): Flush status journal for 1d18b03686124f021810b831ea32e069: 2024-11-14T09:16:26,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:26,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=145 2024-11-14T09:16:26,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4169): Remote procedure done, pid=145 2024-11-14T09:16:26,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,564 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-14T09:16:26,564 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8920 sec 2024-11-14T09:16:26,566 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees in 1.9020 sec 2024-11-14T09:16:26,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-14T09:16:26,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,815 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:16:26,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,816 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:16:26,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-11-14T09:16:26,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-14T09:16:26,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,817 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:16:26,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,818 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:16:26,819 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:16:26,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on 1d18b03686124f021810b831ea32e069 2024-11-14T09:16:26,912 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1d18b03686124f021810b831ea32e069 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T09:16:26,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d18b03686124f021810b831ea32e069, store=A 2024-11-14T09:16:26,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:26,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d18b03686124f021810b831ea32e069, store=B 2024-11-14T09:16:26,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:26,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d18b03686124f021810b831ea32e069, store=C 2024-11-14T09:16:26,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:26,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,921 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411146947e054673d4ba8a964accc4e06edc6_1d18b03686124f021810b831ea32e069 is 50, key is test_row_0/A:col10/1731575786902/Put/seqid=0 2024-11-14T09:16:26,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-14T09:16:26,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742349_1525 (size=24358) 2024-11-14T09:16:26,933 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:26,936 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411146947e054673d4ba8a964accc4e06edc6_1d18b03686124f021810b831ea32e069 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411146947e054673d4ba8a964accc4e06edc6_1d18b03686124f021810b831ea32e069 2024-11-14T09:16:26,937 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/22412f2db5a74503972db6f733710009, store: [table=TestAcidGuarantees family=A region=1d18b03686124f021810b831ea32e069] 2024-11-14T09:16:26,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/22412f2db5a74503972db6f733710009 is 175, key is test_row_0/A:col10/1731575786902/Put/seqid=0 2024-11-14T09:16:26,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742348_1524 (size=73995) 2024-11-14T09:16:26,940 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=60, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/22412f2db5a74503972db6f733710009 2024-11-14T09:16:26,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/33b69277391344499c80bea12591dd72 is 50, key is test_row_0/B:col10/1731575786902/Put/seqid=0 2024-11-14T09:16:26,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:26,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731575846940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:26,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:26,949 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:26,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58162 deadline: 1731575846942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:26,949 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:26,949 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:26,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:26,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58142 deadline: 1731575846942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:26,949 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:26,950 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:26,950 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:26,950 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:26,950 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:26,950 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:26,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:26,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1731575846945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:26,951 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731575786937, max timestamp=1731575786947Num uniques 859; 2024-11-14T09:16:26,951 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:16:26,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:26,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:26,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:26,953 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731575786937, max timestamp=1731575786947Num uniques 859; 2024-11-14T09:16:26,953 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:16:26,955 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731575786937, max timestamp=1731575786947Num uniques 859; 2024-11-14T09:16:26,955 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-14T09:16:26,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742350_1526 (size=12001) 2024-11-14T09:16:26,958 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/33b69277391344499c80bea12591dd72 2024-11-14T09:16:26,965 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/ccfe26afa43442ab99e961a052ca969d is 50, key is test_row_0/C:col10/1731575786902/Put/seqid=0 2024-11-14T09:16:26,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742351_1527 (size=12001) 2024-11-14T09:16:26,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-14T09:16:26,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:26,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. as already flushing 2024-11-14T09:16:26,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:26,971 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:26,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:26,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:27,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:27,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731575847035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:27,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:27,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:27,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:27,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:27,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:27,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731575847055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:27,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58162 deadline: 1731575847055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:27,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:27,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:27,058 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:27,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58142 deadline: 1731575847055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:27,058 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:27,058 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:27,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1731575847055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:27,058 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:27,058 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:27,058 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:27,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:27,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:27,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:27,058 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:27,058 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:27,058 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:27,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-14T09:16:27,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:27,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. as already flushing 2024-11-14T09:16:27,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:27,125 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:27,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:27,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-14T09:16:27,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:27,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731575847145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:27,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:27,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:27,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:27,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:27,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731575847265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:27,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:27,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58162 deadline: 1731575847265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:27,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:27,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:27,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1731575847265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:27,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58142 deadline: 1731575847265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:27,271 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:27,271 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:27,271 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:27,271 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:27,272 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:27,272 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:27,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:27,272 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:27,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:27,272 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:27,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:27,272 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:27,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-14T09:16:27,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:27,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. as already flushing 2024-11-14T09:16:27,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:27,280 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:27,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:27,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:27,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:27,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731575847355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:27,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:27,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:27,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:27,369 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/ccfe26afa43442ab99e961a052ca969d 2024-11-14T09:16:27,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/22412f2db5a74503972db6f733710009 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/22412f2db5a74503972db6f733710009 2024-11-14T09:16:27,376 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/22412f2db5a74503972db6f733710009, entries=400, sequenceid=60, filesize=72.3 K 2024-11-14T09:16:27,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/33b69277391344499c80bea12591dd72 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/33b69277391344499c80bea12591dd72 2024-11-14T09:16:27,380 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/33b69277391344499c80bea12591dd72, entries=150, sequenceid=60, filesize=11.7 K 2024-11-14T09:16:27,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/ccfe26afa43442ab99e961a052ca969d as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/ccfe26afa43442ab99e961a052ca969d 2024-11-14T09:16:27,383 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/ccfe26afa43442ab99e961a052ca969d, entries=150, sequenceid=60, filesize=11.7 K 2024-11-14T09:16:27,384 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.08 KB/130125 for 1d18b03686124f021810b831ea32e069 in 472ms, sequenceid=60, compaction requested=true 2024-11-14T09:16:27,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1d18b03686124f021810b831ea32e069: 2024-11-14T09:16:27,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d18b03686124f021810b831ea32e069:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:16:27,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:16:27,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d18b03686124f021810b831ea32e069:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:16:27,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:16:27,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d18b03686124f021810b831ea32e069:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:16:27,384 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:16:27,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:16:27,384 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:16:27,385 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:16:27,385 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 135905 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:16:27,385 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): 1d18b03686124f021810b831ea32e069/B is initiating minor compaction (all files) 2024-11-14T09:16:27,385 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): 1d18b03686124f021810b831ea32e069/A is initiating minor compaction (all files) 2024-11-14T09:16:27,386 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d18b03686124f021810b831ea32e069/A in TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:27,386 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d18b03686124f021810b831ea32e069/B in TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:27,386 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/cdb74091cb0c43338a5bbaa1a49955d0, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/924d28422b2443b3a0ecc9735ba1c183, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/22412f2db5a74503972db6f733710009] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp, totalSize=132.7 K 2024-11-14T09:16:27,386 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/abb0b3279ea14c4797b2b2a3c1951ff7, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/b1277aac80824eceab1c93889bc13d1d, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/33b69277391344499c80bea12591dd72] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp, totalSize=35.2 K 2024-11-14T09:16:27,386 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:27,386 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. files: [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/cdb74091cb0c43338a5bbaa1a49955d0, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/924d28422b2443b3a0ecc9735ba1c183, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/22412f2db5a74503972db6f733710009] 2024-11-14T09:16:27,386 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting abb0b3279ea14c4797b2b2a3c1951ff7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731575784669 2024-11-14T09:16:27,386 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting cdb74091cb0c43338a5bbaa1a49955d0, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731575784669 2024-11-14T09:16:27,387 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 924d28422b2443b3a0ecc9735ba1c183, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731575784705 2024-11-14T09:16:27,387 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting b1277aac80824eceab1c93889bc13d1d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731575784705 2024-11-14T09:16:27,387 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22412f2db5a74503972db6f733710009, keycount=400, bloomtype=ROW, size=72.3 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731575786869 2024-11-14T09:16:27,387 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 33b69277391344499c80bea12591dd72, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731575786902 2024-11-14T09:16:27,392 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1d18b03686124f021810b831ea32e069] 2024-11-14T09:16:27,394 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d18b03686124f021810b831ea32e069#B#compaction#450 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:16:27,395 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/55b3d9bd72a240859ad7f4748b5f94fd is 50, key is test_row_0/B:col10/1731575786902/Put/seqid=0 2024-11-14T09:16:27,397 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411147c4cbeabd49e4847800f282a02b0e0f4_1d18b03686124f021810b831ea32e069 store=[table=TestAcidGuarantees family=A region=1d18b03686124f021810b831ea32e069] 2024-11-14T09:16:27,400 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411147c4cbeabd49e4847800f282a02b0e0f4_1d18b03686124f021810b831ea32e069, store=[table=TestAcidGuarantees family=A region=1d18b03686124f021810b831ea32e069] 2024-11-14T09:16:27,400 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411147c4cbeabd49e4847800f282a02b0e0f4_1d18b03686124f021810b831ea32e069 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1d18b03686124f021810b831ea32e069] 2024-11-14T09:16:27,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742352_1528 (size=4469) 2024-11-14T09:16:27,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-14T09:16:27,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:27,433 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing 1d18b03686124f021810b831ea32e069 3/3 column families, dataSize=127.08 KB heapSize=364.15 KB 2024-11-14T09:16:27,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d18b03686124f021810b831ea32e069, store=A 2024-11-14T09:16:27,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:16:27,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d18b03686124f021810b831ea32e069, store=B 2024-11-14T09:16:27,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:16:27,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d18b03686124f021810b831ea32e069, store=C 2024-11-14T09:16:27,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:16:27,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742353_1529 (size=12104) 2024-11-14T09:16:27,439 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/55b3d9bd72a240859ad7f4748b5f94fd as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/55b3d9bd72a240859ad7f4748b5f94fd 2024-11-14T09:16:27,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111481352f23c9c140b894a6cf05a53e7326_1d18b03686124f021810b831ea32e069 is 50, key is test_row_0/A:col10/1731575787026/Put/seqid=0 2024-11-14T09:16:27,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742354_1530 (size=12154) 2024-11-14T09:16:27,444 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d18b03686124f021810b831ea32e069/B of 1d18b03686124f021810b831ea32e069 into 55b3d9bd72a240859ad7f4748b5f94fd(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:16:27,444 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d18b03686124f021810b831ea32e069: 2024-11-14T09:16:27,444 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., storeName=1d18b03686124f021810b831ea32e069/B, priority=13, startTime=1731575787384; duration=0sec 2024-11-14T09:16:27,444 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:16:27,444 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d18b03686124f021810b831ea32e069:B 2024-11-14T09:16:27,444 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:16:27,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-14T09:16:27,445 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:16:27,445 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): 1d18b03686124f021810b831ea32e069/C is initiating minor compaction (all files) 2024-11-14T09:16:27,445 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d18b03686124f021810b831ea32e069/C in TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:27,445 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/4f3bf9b5135e4f20b04059ed9aa8c7fa, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/8d41bb25d20245718e2e0075b2c38301, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/ccfe26afa43442ab99e961a052ca969d] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp, totalSize=35.2 K 2024-11-14T09:16:27,446 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f3bf9b5135e4f20b04059ed9aa8c7fa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731575784669 2024-11-14T09:16:27,446 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d41bb25d20245718e2e0075b2c38301, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731575784705 2024-11-14T09:16:27,446 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting ccfe26afa43442ab99e961a052ca969d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731575786902 2024-11-14T09:16:27,453 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d18b03686124f021810b831ea32e069#C#compaction#453 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:16:27,453 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/ad2589148e8d4c939b5c46acf98b6a32 is 50, key is test_row_0/C:col10/1731575786902/Put/seqid=0 2024-11-14T09:16:27,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742355_1531 (size=12104) 2024-11-14T09:16:27,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on 1d18b03686124f021810b831ea32e069 2024-11-14T09:16:27,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. as already flushing 2024-11-14T09:16:27,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:27,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731575847584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:27,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:27,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58142 deadline: 1731575847585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:27,590 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:27,590 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:27,590 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:27,590 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:27,590 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:27,590 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:27,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:27,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:27,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58162 deadline: 1731575847586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:27,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1731575847587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:27,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:27,593 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:27,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:27,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:27,593 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:27,593 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:27,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:27,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731575847665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:27,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:27,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:27,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:27,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:27,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731575847695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:27,700 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:27,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:27,700 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:27,700 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:27,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58142 deadline: 1731575847695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:27,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:27,700 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:27,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1731575847695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:27,700 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:27,700 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:27,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:27,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:27,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:27,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:27,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58162 deadline: 1731575847695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:27,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:27,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:27,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:27,827 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d18b03686124f021810b831ea32e069#A#compaction#451 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:16:27,827 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/ce0647ea93ed4f409af189067f47d9d8 is 175, key is test_row_0/A:col10/1731575786902/Put/seqid=0 2024-11-14T09:16:27,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742356_1532 (size=31058) 2024-11-14T09:16:27,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:27,847 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111481352f23c9c140b894a6cf05a53e7326_1d18b03686124f021810b831ea32e069 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111481352f23c9c140b894a6cf05a53e7326_1d18b03686124f021810b831ea32e069 2024-11-14T09:16:27,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/221464a389b240e0981ba6864d45c130, store: [table=TestAcidGuarantees family=A region=1d18b03686124f021810b831ea32e069] 2024-11-14T09:16:27,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/221464a389b240e0981ba6864d45c130 is 175, key is test_row_0/A:col10/1731575787026/Put/seqid=0 2024-11-14T09:16:27,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742357_1533 (size=30955) 2024-11-14T09:16:27,853 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=96, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/221464a389b240e0981ba6864d45c130 2024-11-14T09:16:27,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/e852e42d6c8b4dc5823ff8b289427ee5 is 50, key is test_row_0/B:col10/1731575787026/Put/seqid=0 2024-11-14T09:16:27,863 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/ad2589148e8d4c939b5c46acf98b6a32 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/ad2589148e8d4c939b5c46acf98b6a32 2024-11-14T09:16:27,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742358_1534 (size=12001) 2024-11-14T09:16:27,865 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/e852e42d6c8b4dc5823ff8b289427ee5 2024-11-14T09:16:27,869 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d18b03686124f021810b831ea32e069/C of 1d18b03686124f021810b831ea32e069 into ad2589148e8d4c939b5c46acf98b6a32(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:16:27,869 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d18b03686124f021810b831ea32e069: 2024-11-14T09:16:27,869 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., storeName=1d18b03686124f021810b831ea32e069/C, priority=13, startTime=1731575787384; duration=0sec 2024-11-14T09:16:27,870 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:16:27,870 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d18b03686124f021810b831ea32e069:C 2024-11-14T09:16:27,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/ee8b9b11c5814d3bbc1a3d54b4ac8a2b is 50, key is test_row_0/C:col10/1731575787026/Put/seqid=0 2024-11-14T09:16:27,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742359_1535 (size=12001) 2024-11-14T09:16:27,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:27,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731575847905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:27,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:27,908 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:27,908 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:27,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1731575847905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:27,908 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:27,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:27,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:27,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:27,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:27,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58142 deadline: 1731575847905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:27,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:27,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:27,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:27,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58162 deadline: 1731575847905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:27,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:27,909 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:27,909 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:27,909 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:27,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-14T09:16:28,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:28,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731575848174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:28,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:28,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:28,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:28,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:28,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731575848215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:28,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:28,219 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:28,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1731575848215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:28,219 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:28,219 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:28,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:28,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:28,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:28,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58142 deadline: 1731575848215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:28,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:28,219 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:28,219 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:28,219 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:28,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:28,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58162 deadline: 1731575848215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:28,220 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:28,220 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:28,220 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:28,235 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/ce0647ea93ed4f409af189067f47d9d8 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/ce0647ea93ed4f409af189067f47d9d8 2024-11-14T09:16:28,240 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d18b03686124f021810b831ea32e069/A of 1d18b03686124f021810b831ea32e069 into ce0647ea93ed4f409af189067f47d9d8(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:16:28,240 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d18b03686124f021810b831ea32e069: 2024-11-14T09:16:28,240 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., storeName=1d18b03686124f021810b831ea32e069/A, priority=13, startTime=1731575787384; duration=0sec 2024-11-14T09:16:28,240 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:16:28,240 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d18b03686124f021810b831ea32e069:A 2024-11-14T09:16:28,282 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.09 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/ee8b9b11c5814d3bbc1a3d54b4ac8a2b 2024-11-14T09:16:28,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/221464a389b240e0981ba6864d45c130 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/221464a389b240e0981ba6864d45c130 2024-11-14T09:16:28,289 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/221464a389b240e0981ba6864d45c130, entries=150, sequenceid=96, filesize=30.2 K 2024-11-14T09:16:28,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/e852e42d6c8b4dc5823ff8b289427ee5 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/e852e42d6c8b4dc5823ff8b289427ee5 2024-11-14T09:16:28,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,293 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/e852e42d6c8b4dc5823ff8b289427ee5, entries=150, sequenceid=96, filesize=11.7 K 2024-11-14T09:16:28,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/ee8b9b11c5814d3bbc1a3d54b4ac8a2b as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/ee8b9b11c5814d3bbc1a3d54b4ac8a2b 2024-11-14T09:16:28,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,297 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/ee8b9b11c5814d3bbc1a3d54b4ac8a2b, entries=150, sequenceid=96, filesize=11.7 K 2024-11-14T09:16:28,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,298 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~127.08 KB/130125, heapSize ~364.10 KB/372840, currentSize=53.67 KB/54960 for 1d18b03686124f021810b831ea32e069 in 866ms, sequenceid=96, compaction requested=false 2024-11-14T09:16:28,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for 1d18b03686124f021810b831ea32e069: 2024-11-14T09:16:28,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:28,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-11-14T09:16:28,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-11-14T09:16:28,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,302 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-14T09:16:28,302 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4810 sec 2024-11-14T09:16:28,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,303 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 1.4860 sec 2024-11-14T09:16:28,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on 1d18b03686124f021810b831ea32e069 2024-11-14T09:16:28,732 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1d18b03686124f021810b831ea32e069 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-14T09:16:28,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d18b03686124f021810b831ea32e069, store=A 2024-11-14T09:16:28,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:28,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d18b03686124f021810b831ea32e069, store=B 2024-11-14T09:16:28,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:28,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d18b03686124f021810b831ea32e069, store=C 2024-11-14T09:16:28,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:28,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,741 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411145a96e1c2d40044ddb85b2b480659cbb6_1d18b03686124f021810b831ea32e069 is 50, key is test_row_0/A:col10/1731575788730/Put/seqid=0 2024-11-14T09:16:28,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742360_1536 (size=12154) 2024-11-14T09:16:28,759 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:28,764 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411145a96e1c2d40044ddb85b2b480659cbb6_1d18b03686124f021810b831ea32e069 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145a96e1c2d40044ddb85b2b480659cbb6_1d18b03686124f021810b831ea32e069 2024-11-14T09:16:28,765 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/163956fbdef6422486d02fda8b5ecb91, store: [table=TestAcidGuarantees family=A region=1d18b03686124f021810b831ea32e069] 2024-11-14T09:16:28,766 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/163956fbdef6422486d02fda8b5ecb91 is 175, key is test_row_0/A:col10/1731575788730/Put/seqid=0 2024-11-14T09:16:28,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742361_1537 (size=30951) 2024-11-14T09:16:28,778 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=111, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/163956fbdef6422486d02fda8b5ecb91 2024-11-14T09:16:28,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:28,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58162 deadline: 1731575848771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:28,781 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731575788761, max timestamp=1731575788777Num uniques 859; 2024-11-14T09:16:28,781 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:16:28,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731575788761, max timestamp=1731575788777Num uniques 859; 2024-11-14T09:16:28,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-14T09:16:28,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731575788761, max timestamp=1731575788777Num uniques 859; 2024-11-14T09:16:28,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/83f56b55f2af:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-14T09:16:28,783 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:28,783 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:28,783 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:28,790 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/cc42c32a131b402b93887ef1e5998ad2 is 50, key is test_row_0/B:col10/1731575788730/Put/seqid=0 2024-11-14T09:16:28,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:28,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58142 deadline: 1731575848812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:28,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:28,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731575848814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:28,819 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:28,819 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:28,819 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:28,819 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:28,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:28,819 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:28,819 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:28,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1731575848815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:28,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:28,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:28,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:28,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742362_1538 (size=9657) 2024-11-14T09:16:28,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:28,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58162 deadline: 1731575848885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:28,888 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:28,888 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:28,888 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:28,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:28,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1731575848925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:28,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:28,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58142 deadline: 1731575848925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:28,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:28,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:28,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:28,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:28,930 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:28,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731575848925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:28,930 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:28,931 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:28,931 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:28,931 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:28,931 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:28,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-14T09:16:28,965 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:16:28,965 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:16:28,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-14T09:16:28,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-14T09:16:28,967 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:16:28,968 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:16:28,968 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:16:29,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-14T09:16:29,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:29,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58162 deadline: 1731575849095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:29,098 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:29,098 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:29,098 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:29,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-14T09:16:29,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:29,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. as already flushing 2024-11-14T09:16:29,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:29,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:29,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:29,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:29,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:29,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1731575849134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:29,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:29,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:29,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58142 deadline: 1731575849134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:29,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:29,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:29,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:29,139 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:29,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731575849135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:29,139 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:29,139 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:29,139 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:29,140 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:29,140 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:29,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:29,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731575849185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:29,188 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_2' in 1d18b03686124f021810b831ea32e069 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:16:29,189 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:29,189 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:29,189 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:29,228 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/cc42c32a131b402b93887ef1e5998ad2 2024-11-14T09:16:29,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/3313320d349d453198100d89ec9ce8c1 is 50, key is test_row_0/C:col10/1731575788730/Put/seqid=0 2024-11-14T09:16:29,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742363_1539 (size=9657) 2024-11-14T09:16:29,241 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/3313320d349d453198100d89ec9ce8c1 2024-11-14T09:16:29,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/163956fbdef6422486d02fda8b5ecb91 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/163956fbdef6422486d02fda8b5ecb91 2024-11-14T09:16:29,249 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/163956fbdef6422486d02fda8b5ecb91, entries=150, sequenceid=111, filesize=30.2 K 2024-11-14T09:16:29,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/cc42c32a131b402b93887ef1e5998ad2 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/cc42c32a131b402b93887ef1e5998ad2 2024-11-14T09:16:29,255 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/cc42c32a131b402b93887ef1e5998ad2, entries=100, sequenceid=111, filesize=9.4 K 2024-11-14T09:16:29,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/3313320d349d453198100d89ec9ce8c1 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/3313320d349d453198100d89ec9ce8c1 2024-11-14T09:16:29,259 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/3313320d349d453198100d89ec9ce8c1, entries=100, sequenceid=111, filesize=9.4 K 2024-11-14T09:16:29,260 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=119.97 KB/122850 for 1d18b03686124f021810b831ea32e069 in 528ms, sequenceid=111, compaction requested=true 2024-11-14T09:16:29,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1d18b03686124f021810b831ea32e069: 2024-11-14T09:16:29,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d18b03686124f021810b831ea32e069:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:16:29,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:16:29,260 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:16:29,260 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:16:29,261 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92964 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:16:29,261 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): 1d18b03686124f021810b831ea32e069/A is initiating minor compaction (all files) 2024-11-14T09:16:29,261 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d18b03686124f021810b831ea32e069/A in TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:29,261 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/ce0647ea93ed4f409af189067f47d9d8, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/221464a389b240e0981ba6864d45c130, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/163956fbdef6422486d02fda8b5ecb91] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp, totalSize=90.8 K 2024-11-14T09:16:29,262 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:29,262 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. files: [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/ce0647ea93ed4f409af189067f47d9d8, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/221464a389b240e0981ba6864d45c130, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/163956fbdef6422486d02fda8b5ecb91] 2024-11-14T09:16:29,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d18b03686124f021810b831ea32e069:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:16:29,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:16:29,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d18b03686124f021810b831ea32e069:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:16:29,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:16:29,262 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce0647ea93ed4f409af189067f47d9d8, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731575786902 2024-11-14T09:16:29,262 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:16:29,262 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 221464a389b240e0981ba6864d45c130, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731575786999 2024-11-14T09:16:29,263 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): 1d18b03686124f021810b831ea32e069/B is initiating minor compaction (all files) 2024-11-14T09:16:29,263 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d18b03686124f021810b831ea32e069/B in TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:29,263 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/55b3d9bd72a240859ad7f4748b5f94fd, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/e852e42d6c8b4dc5823ff8b289427ee5, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/cc42c32a131b402b93887ef1e5998ad2] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp, totalSize=33.0 K 2024-11-14T09:16:29,263 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 55b3d9bd72a240859ad7f4748b5f94fd, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731575786902 2024-11-14T09:16:29,263 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 163956fbdef6422486d02fda8b5ecb91, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1731575787583 2024-11-14T09:16:29,263 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting e852e42d6c8b4dc5823ff8b289427ee5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731575786999 2024-11-14T09:16:29,264 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting cc42c32a131b402b93887ef1e5998ad2, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1731575787586 2024-11-14T09:16:29,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-14T09:16:29,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:29,274 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing 1d18b03686124f021810b831ea32e069 3/3 column families, dataSize=119.97 KB heapSize=343.31 KB 2024-11-14T09:16:29,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d18b03686124f021810b831ea32e069, store=A 2024-11-14T09:16:29,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:16:29,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d18b03686124f021810b831ea32e069, store=B 2024-11-14T09:16:29,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:16:29,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d18b03686124f021810b831ea32e069, store=C 2024-11-14T09:16:29,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:16:29,277 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d18b03686124f021810b831ea32e069#B#compaction#459 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:16:29,278 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/82f827e7937d44ebbc0813a6037841e6 is 50, key is test_row_0/B:col10/1731575788730/Put/seqid=0 2024-11-14T09:16:29,279 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1d18b03686124f021810b831ea32e069] 2024-11-14T09:16:29,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-14T09:16:29,297 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114e8df5c6a02b049d683757cd1ffcc2fb0_1d18b03686124f021810b831ea32e069 store=[table=TestAcidGuarantees family=A region=1d18b03686124f021810b831ea32e069] 2024-11-14T09:16:29,299 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114e8df5c6a02b049d683757cd1ffcc2fb0_1d18b03686124f021810b831ea32e069, store=[table=TestAcidGuarantees family=A region=1d18b03686124f021810b831ea32e069] 2024-11-14T09:16:29,299 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e8df5c6a02b049d683757cd1ffcc2fb0_1d18b03686124f021810b831ea32e069 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1d18b03686124f021810b831ea32e069] 2024-11-14T09:16:29,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742364_1540 (size=12207) 2024-11-14T09:16:29,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411148fd42af643ab400ebba6b6db010bf9d2_1d18b03686124f021810b831ea32e069 is 50, key is test_row_0/A:col10/1731575788805/Put/seqid=0 2024-11-14T09:16:29,320 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/82f827e7937d44ebbc0813a6037841e6 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/82f827e7937d44ebbc0813a6037841e6 2024-11-14T09:16:29,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742365_1541 (size=4469) 2024-11-14T09:16:29,326 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d18b03686124f021810b831ea32e069#A#compaction#460 average throughput is 0.52 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:16:29,326 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/778813b64a414e6198ed7beba49f09e7 is 175, key is test_row_0/A:col10/1731575788730/Put/seqid=0 2024-11-14T09:16:29,328 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d18b03686124f021810b831ea32e069/B of 1d18b03686124f021810b831ea32e069 into 82f827e7937d44ebbc0813a6037841e6(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:16:29,328 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d18b03686124f021810b831ea32e069: 2024-11-14T09:16:29,328 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., storeName=1d18b03686124f021810b831ea32e069/B, priority=13, startTime=1731575789260; duration=0sec 2024-11-14T09:16:29,329 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:16:29,329 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d18b03686124f021810b831ea32e069:B 2024-11-14T09:16:29,329 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:16:29,330 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:16:29,331 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): 1d18b03686124f021810b831ea32e069/C is initiating minor compaction (all files) 2024-11-14T09:16:29,331 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d18b03686124f021810b831ea32e069/C in TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:29,331 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/ad2589148e8d4c939b5c46acf98b6a32, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/ee8b9b11c5814d3bbc1a3d54b4ac8a2b, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/3313320d349d453198100d89ec9ce8c1] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp, totalSize=33.0 K 2024-11-14T09:16:29,331 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting ad2589148e8d4c939b5c46acf98b6a32, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731575786902 2024-11-14T09:16:29,333 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting ee8b9b11c5814d3bbc1a3d54b4ac8a2b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731575786999 2024-11-14T09:16:29,333 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 3313320d349d453198100d89ec9ce8c1, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1731575787586 2024-11-14T09:16:29,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742366_1542 (size=12304) 2024-11-14T09:16:29,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742367_1543 (size=31268) 2024-11-14T09:16:29,342 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d18b03686124f021810b831ea32e069#C#compaction#462 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:16:29,343 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/98b4639f19fd48a195cef8d62878dcf7 is 50, key is test_row_0/C:col10/1731575788730/Put/seqid=0 2024-11-14T09:16:29,344 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/778813b64a414e6198ed7beba49f09e7 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/778813b64a414e6198ed7beba49f09e7 2024-11-14T09:16:29,353 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d18b03686124f021810b831ea32e069/A of 1d18b03686124f021810b831ea32e069 into 778813b64a414e6198ed7beba49f09e7(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:16:29,353 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d18b03686124f021810b831ea32e069: 2024-11-14T09:16:29,353 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., storeName=1d18b03686124f021810b831ea32e069/A, priority=13, startTime=1731575789260; duration=0sec 2024-11-14T09:16:29,353 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:16:29,353 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d18b03686124f021810b831ea32e069:A 2024-11-14T09:16:29,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742368_1544 (size=12207) 2024-11-14T09:16:29,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on 1d18b03686124f021810b831ea32e069 2024-11-14T09:16:29,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. as already flushing 2024-11-14T09:16:29,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:29,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731575849445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:29,450 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:29,450 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:29,450 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:29,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:29,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58162 deadline: 1731575849447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:29,454 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:29,454 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:29,454 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:29,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:29,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1731575849449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:29,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:29,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:29,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:29,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58142 deadline: 1731575849450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:29,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:29,455 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:29,455 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:29,455 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:29,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:29,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58162 deadline: 1731575849565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:29,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:29,570 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:29,570 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:29,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1731575849565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:29,570 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:29,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:29,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:29,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:29,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:29,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58142 deadline: 1731575849565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:29,570 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:29,570 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:29,570 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:29,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-14T09:16:29,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:29,739 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411148fd42af643ab400ebba6b6db010bf9d2_1d18b03686124f021810b831ea32e069 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411148fd42af643ab400ebba6b6db010bf9d2_1d18b03686124f021810b831ea32e069 2024-11-14T09:16:29,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/19893271ff0640daa58ffe84312b425d, store: [table=TestAcidGuarantees family=A region=1d18b03686124f021810b831ea32e069] 2024-11-14T09:16:29,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/19893271ff0640daa58ffe84312b425d is 175, key is test_row_0/A:col10/1731575788805/Put/seqid=0 2024-11-14T09:16:29,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742369_1545 (size=31105) 2024-11-14T09:16:29,778 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/98b4639f19fd48a195cef8d62878dcf7 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/98b4639f19fd48a195cef8d62878dcf7 2024-11-14T09:16:29,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:29,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58162 deadline: 1731575849775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:29,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:29,780 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:29,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1731575849775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:29,780 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:29,780 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:29,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:29,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:29,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:29,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:29,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58142 deadline: 1731575849775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:29,781 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:29,781 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:29,781 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:29,782 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d18b03686124f021810b831ea32e069/C of 1d18b03686124f021810b831ea32e069 into 98b4639f19fd48a195cef8d62878dcf7(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:16:29,782 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d18b03686124f021810b831ea32e069: 2024-11-14T09:16:29,782 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., storeName=1d18b03686124f021810b831ea32e069/C, priority=13, startTime=1731575789262; duration=0sec 2024-11-14T09:16:29,783 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:16:29,783 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d18b03686124f021810b831ea32e069:C 2024-11-14T09:16:29,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:29,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731575849955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:29,958 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:29,958 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:29,958 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:30,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:30,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58162 deadline: 1731575850085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:30,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:30,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1731575850085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:30,089 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:30,089 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:30,089 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:30,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:30,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58142 deadline: 1731575850085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:30,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:30,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:30,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:30,089 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:30,089 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:30,089 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:30,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-14T09:16:30,144 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=146, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/19893271ff0640daa58ffe84312b425d 2024-11-14T09:16:30,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/ff7c1be6866541c0ad90fa5b0e261ae5 is 50, key is test_row_0/B:col10/1731575788805/Put/seqid=0 2024-11-14T09:16:30,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742370_1546 (size=12151) 2024-11-14T09:16:30,555 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=39.86 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/ff7c1be6866541c0ad90fa5b0e261ae5 2024-11-14T09:16:30,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/5c55161fd10e4bd5a82bacc8faba0bb4 is 50, key is test_row_0/C:col10/1731575788805/Put/seqid=0 2024-11-14T09:16:30,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742371_1547 (size=12151) 2024-11-14T09:16:30,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:30,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58162 deadline: 1731575850595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:30,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:30,599 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:30,599 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:30,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:30,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1731575850595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:30,599 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:30,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58142 deadline: 1731575850595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:30,600 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:30,600 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:30,600 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:30,600 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:30,600 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:30,600 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:30,966 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=39.86 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/5c55161fd10e4bd5a82bacc8faba0bb4 2024-11-14T09:16:30,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:30,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731575850965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:30,969 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_0' in 1d18b03686124f021810b831ea32e069 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:16:30,969 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:30,969 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:30,969 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:30,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/19893271ff0640daa58ffe84312b425d as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/19893271ff0640daa58ffe84312b425d 2024-11-14T09:16:30,975 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/19893271ff0640daa58ffe84312b425d, entries=150, sequenceid=146, filesize=30.4 K 2024-11-14T09:16:30,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/ff7c1be6866541c0ad90fa5b0e261ae5 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/ff7c1be6866541c0ad90fa5b0e261ae5 2024-11-14T09:16:30,979 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/ff7c1be6866541c0ad90fa5b0e261ae5, entries=150, sequenceid=146, filesize=11.9 K 2024-11-14T09:16:30,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/5c55161fd10e4bd5a82bacc8faba0bb4 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/5c55161fd10e4bd5a82bacc8faba0bb4 2024-11-14T09:16:30,982 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/5c55161fd10e4bd5a82bacc8faba0bb4, entries=150, sequenceid=146, filesize=11.9 K 2024-11-14T09:16:30,983 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~119.97 KB/122850, heapSize ~343.27 KB/351504, currentSize=60.38 KB/61830 for 1d18b03686124f021810b831ea32e069 in 1709ms, sequenceid=146, compaction requested=false 2024-11-14T09:16:30,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for 1d18b03686124f021810b831ea32e069: 2024-11-14T09:16:30,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:30,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-14T09:16:30,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-14T09:16:30,986 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-14T09:16:30,986 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0160 sec 2024-11-14T09:16:30,988 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 2.0210 sec 2024-11-14T09:16:31,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-14T09:16:31,115 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:16:31,115 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:16:31,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-14T09:16:31,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-14T09:16:31,118 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:16:31,118 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:16:31,118 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:16:31,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on 1d18b03686124f021810b831ea32e069 2024-11-14T09:16:31,198 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1d18b03686124f021810b831ea32e069 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-14T09:16:31,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d18b03686124f021810b831ea32e069, store=A 2024-11-14T09:16:31,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:31,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d18b03686124f021810b831ea32e069, store=B 2024-11-14T09:16:31,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:31,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d18b03686124f021810b831ea32e069, store=C 2024-11-14T09:16:31,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:31,204 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411140d0a30f8389843838ad922a791f2e8b8_1d18b03686124f021810b831ea32e069 is 50, key is test_row_0/A:col10/1731575789449/Put/seqid=0 2024-11-14T09:16:31,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742372_1548 (size=14794) 2024-11-14T09:16:31,208 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:31,212 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411140d0a30f8389843838ad922a791f2e8b8_1d18b03686124f021810b831ea32e069 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411140d0a30f8389843838ad922a791f2e8b8_1d18b03686124f021810b831ea32e069 2024-11-14T09:16:31,213 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/926838a9c6bb447fb1e088fbe7d0c918, store: [table=TestAcidGuarantees family=A region=1d18b03686124f021810b831ea32e069] 2024-11-14T09:16:31,213 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/926838a9c6bb447fb1e088fbe7d0c918 is 175, key is test_row_0/A:col10/1731575789449/Put/seqid=0 2024-11-14T09:16:31,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742373_1549 (size=39749) 2024-11-14T09:16:31,217 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=162, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/926838a9c6bb447fb1e088fbe7d0c918 2024-11-14T09:16:31,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/70c05adab43340ce88cdc1aaca83f793 is 50, key is test_row_0/B:col10/1731575789449/Put/seqid=0 2024-11-14T09:16:31,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-14T09:16:31,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742374_1550 (size=12151) 2024-11-14T09:16:31,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-14T09:16:31,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:31,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. as already flushing 2024-11-14T09:16:31,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:31,271 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:31,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:31,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:31,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:31,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731575851276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:31,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:31,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:31,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:31,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:31,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731575851385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:31,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:31,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:31,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:31,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-14T09:16:31,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:31,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. as already flushing 2024-11-14T09:16:31,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:31,423 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:31,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:31,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:31,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-14T09:16:31,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-14T09:16:31,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:31,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. as already flushing 2024-11-14T09:16:31,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:31,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:31,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:31,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:31,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:31,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731575851594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:31,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:31,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:31,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:31,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:31,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58162 deadline: 1731575851604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:31,607 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_1' in 1d18b03686124f021810b831ea32e069 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:16:31,608 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:31,608 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:31,608 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:31,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:31,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1731575851615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:31,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:31,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58142 deadline: 1731575851615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:31,619 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_0' in 1d18b03686124f021810b831ea32e069 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:16:31,619 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:31,619 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:31,619 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:31,619 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_1' in 1d18b03686124f021810b831ea32e069 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:16:31,619 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:31,619 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:31,619 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:31,627 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/70c05adab43340ce88cdc1aaca83f793 2024-11-14T09:16:31,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/1cb2c63c4888404d8e0b5ff435f300c0 is 50, key is test_row_0/C:col10/1731575789449/Put/seqid=0 2024-11-14T09:16:31,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742375_1551 (size=12151) 2024-11-14T09:16:31,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-14T09:16:31,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:31,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. as already flushing 2024-11-14T09:16:31,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:31,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:31,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:31,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:31,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-14T09:16:31,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-14T09:16:31,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:31,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. as already flushing 2024-11-14T09:16:31,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:31,882 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:31,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:31,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:31,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:31,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731575851905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:31,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:31,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:31,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:32,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-14T09:16:32,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:32,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. as already flushing 2024-11-14T09:16:32,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:32,036 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:32,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:32,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:32,044 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/1cb2c63c4888404d8e0b5ff435f300c0 2024-11-14T09:16:32,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/926838a9c6bb447fb1e088fbe7d0c918 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/926838a9c6bb447fb1e088fbe7d0c918 2024-11-14T09:16:32,051 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/926838a9c6bb447fb1e088fbe7d0c918, entries=200, sequenceid=162, filesize=38.8 K 2024-11-14T09:16:32,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/70c05adab43340ce88cdc1aaca83f793 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/70c05adab43340ce88cdc1aaca83f793 2024-11-14T09:16:32,055 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/70c05adab43340ce88cdc1aaca83f793, entries=150, sequenceid=162, filesize=11.9 K 2024-11-14T09:16:32,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/1cb2c63c4888404d8e0b5ff435f300c0 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/1cb2c63c4888404d8e0b5ff435f300c0 2024-11-14T09:16:32,059 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/1cb2c63c4888404d8e0b5ff435f300c0, entries=150, sequenceid=162, filesize=11.9 K 2024-11-14T09:16:32,059 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 1d18b03686124f021810b831ea32e069 in 862ms, sequenceid=162, compaction requested=true 2024-11-14T09:16:32,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1d18b03686124f021810b831ea32e069: 2024-11-14T09:16:32,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d18b03686124f021810b831ea32e069:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:16:32,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:16:32,060 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:16:32,060 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:16:32,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d18b03686124f021810b831ea32e069:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:16:32,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:16:32,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d18b03686124f021810b831ea32e069:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:16:32,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:16:32,061 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:16:32,061 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102122 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:16:32,061 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): 1d18b03686124f021810b831ea32e069/A is initiating minor compaction (all files) 2024-11-14T09:16:32,061 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d18b03686124f021810b831ea32e069/A in TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:32,061 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/778813b64a414e6198ed7beba49f09e7, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/19893271ff0640daa58ffe84312b425d, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/926838a9c6bb447fb1e088fbe7d0c918] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp, totalSize=99.7 K 2024-11-14T09:16:32,061 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:32,061 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. files: [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/778813b64a414e6198ed7beba49f09e7, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/19893271ff0640daa58ffe84312b425d, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/926838a9c6bb447fb1e088fbe7d0c918] 2024-11-14T09:16:32,061 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): 1d18b03686124f021810b831ea32e069/B is initiating minor compaction (all files) 2024-11-14T09:16:32,061 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d18b03686124f021810b831ea32e069/B in TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:32,061 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/82f827e7937d44ebbc0813a6037841e6, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/ff7c1be6866541c0ad90fa5b0e261ae5, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/70c05adab43340ce88cdc1aaca83f793] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp, totalSize=35.7 K 2024-11-14T09:16:32,062 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 82f827e7937d44ebbc0813a6037841e6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1731575786999 2024-11-14T09:16:32,062 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 778813b64a414e6198ed7beba49f09e7, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1731575786999 2024-11-14T09:16:32,062 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting ff7c1be6866541c0ad90fa5b0e261ae5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731575788805 2024-11-14T09:16:32,062 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19893271ff0640daa58ffe84312b425d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731575788805 2024-11-14T09:16:32,062 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 70c05adab43340ce88cdc1aaca83f793, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1731575789445 2024-11-14T09:16:32,062 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 926838a9c6bb447fb1e088fbe7d0c918, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1731575789436 2024-11-14T09:16:32,067 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1d18b03686124f021810b831ea32e069] 2024-11-14T09:16:32,068 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d18b03686124f021810b831ea32e069#B#compaction#468 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:16:32,069 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/b00111888185487c94bbd140cf7406a7 is 50, key is test_row_0/B:col10/1731575789449/Put/seqid=0 2024-11-14T09:16:32,070 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114218643aecc73486eb1ce767e06239cf6_1d18b03686124f021810b831ea32e069 store=[table=TestAcidGuarantees family=A region=1d18b03686124f021810b831ea32e069] 2024-11-14T09:16:32,072 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114218643aecc73486eb1ce767e06239cf6_1d18b03686124f021810b831ea32e069, store=[table=TestAcidGuarantees family=A region=1d18b03686124f021810b831ea32e069] 2024-11-14T09:16:32,072 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114218643aecc73486eb1ce767e06239cf6_1d18b03686124f021810b831ea32e069 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1d18b03686124f021810b831ea32e069] 2024-11-14T09:16:32,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742377_1553 (size=4469) 2024-11-14T09:16:32,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742376_1552 (size=12459) 2024-11-14T09:16:32,091 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d18b03686124f021810b831ea32e069#A#compaction#469 average throughput is 1.02 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:16:32,092 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/c12bb0e24fa24ca988d909d9d5b457f2 is 175, key is test_row_0/A:col10/1731575789449/Put/seqid=0 2024-11-14T09:16:32,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742378_1554 (size=31413) 2024-11-14T09:16:32,098 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/c12bb0e24fa24ca988d909d9d5b457f2 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/c12bb0e24fa24ca988d909d9d5b457f2 2024-11-14T09:16:32,109 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d18b03686124f021810b831ea32e069/A of 1d18b03686124f021810b831ea32e069 into c12bb0e24fa24ca988d909d9d5b457f2(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:16:32,109 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d18b03686124f021810b831ea32e069: 2024-11-14T09:16:32,109 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., storeName=1d18b03686124f021810b831ea32e069/A, priority=13, startTime=1731575792060; duration=0sec 2024-11-14T09:16:32,109 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:16:32,109 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d18b03686124f021810b831ea32e069:A 2024-11-14T09:16:32,109 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:16:32,110 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:16:32,110 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): 1d18b03686124f021810b831ea32e069/C is initiating minor compaction (all files) 2024-11-14T09:16:32,110 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d18b03686124f021810b831ea32e069/C in TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:32,110 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/98b4639f19fd48a195cef8d62878dcf7, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/5c55161fd10e4bd5a82bacc8faba0bb4, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/1cb2c63c4888404d8e0b5ff435f300c0] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp, totalSize=35.7 K 2024-11-14T09:16:32,111 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98b4639f19fd48a195cef8d62878dcf7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1731575786999 2024-11-14T09:16:32,111 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c55161fd10e4bd5a82bacc8faba0bb4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731575788805 2024-11-14T09:16:32,111 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1cb2c63c4888404d8e0b5ff435f300c0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1731575789445 2024-11-14T09:16:32,115 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-14T09:16:32,115 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-14T09:16:32,118 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d18b03686124f021810b831ea32e069#C#compaction#470 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:16:32,118 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/54b3756177fd4d8281dc31d657ef607d is 50, key is test_row_0/C:col10/1731575789449/Put/seqid=0 2024-11-14T09:16:32,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742379_1555 (size=12459) 2024-11-14T09:16:32,130 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/54b3756177fd4d8281dc31d657ef607d as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/54b3756177fd4d8281dc31d657ef607d 2024-11-14T09:16:32,135 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d18b03686124f021810b831ea32e069/C of 1d18b03686124f021810b831ea32e069 into 54b3756177fd4d8281dc31d657ef607d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:16:32,135 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d18b03686124f021810b831ea32e069: 2024-11-14T09:16:32,135 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., storeName=1d18b03686124f021810b831ea32e069/C, priority=13, startTime=1731575792060; duration=0sec 2024-11-14T09:16:32,135 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:16:32,135 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d18b03686124f021810b831ea32e069:C 2024-11-14T09:16:32,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-14T09:16:32,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:32,189 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 1d18b03686124f021810b831ea32e069 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-14T09:16:32,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d18b03686124f021810b831ea32e069, store=A 2024-11-14T09:16:32,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:32,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d18b03686124f021810b831ea32e069, store=B 2024-11-14T09:16:32,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:32,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d18b03686124f021810b831ea32e069, store=C 2024-11-14T09:16:32,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:32,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111478c04de1794247b196464d6326b6a584_1d18b03686124f021810b831ea32e069 is 50, key is test_row_0/A:col10/1731575791264/Put/seqid=0 2024-11-14T09:16:32,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742380_1556 (size=12304) 2024-11-14T09:16:32,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:32,202 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111478c04de1794247b196464d6326b6a584_1d18b03686124f021810b831ea32e069 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111478c04de1794247b196464d6326b6a584_1d18b03686124f021810b831ea32e069 2024-11-14T09:16:32,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/ed08a05d99ca40769f9b6bb4e4d0271f, store: [table=TestAcidGuarantees family=A region=1d18b03686124f021810b831ea32e069] 2024-11-14T09:16:32,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/ed08a05d99ca40769f9b6bb4e4d0271f is 175, key is test_row_0/A:col10/1731575791264/Put/seqid=0 2024-11-14T09:16:32,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742381_1557 (size=31105) 2024-11-14T09:16:32,210 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=183, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/ed08a05d99ca40769f9b6bb4e4d0271f 2024-11-14T09:16:32,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/2709a094ac9b461c9aca8d4017c244ce is 50, key is test_row_0/B:col10/1731575791264/Put/seqid=0 2024-11-14T09:16:32,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742382_1558 (size=12151) 2024-11-14T09:16:32,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-14T09:16:32,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on 1d18b03686124f021810b831ea32e069 2024-11-14T09:16:32,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. as already flushing 2024-11-14T09:16:32,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:32,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731575852453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:32,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:32,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:32,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:32,496 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/b00111888185487c94bbd140cf7406a7 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/b00111888185487c94bbd140cf7406a7 2024-11-14T09:16:32,500 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d18b03686124f021810b831ea32e069/B of 1d18b03686124f021810b831ea32e069 into b00111888185487c94bbd140cf7406a7(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:16:32,500 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d18b03686124f021810b831ea32e069: 2024-11-14T09:16:32,500 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., storeName=1d18b03686124f021810b831ea32e069/B, priority=13, startTime=1731575792060; duration=0sec 2024-11-14T09:16:32,500 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:16:32,500 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d18b03686124f021810b831ea32e069:B 2024-11-14T09:16:32,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:32,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731575852565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:32,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:32,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:32,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:32,631 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/2709a094ac9b461c9aca8d4017c244ce 2024-11-14T09:16:32,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/d54e098e94ef4e45a1f1eb3e9f5dbe3c is 50, key is test_row_0/C:col10/1731575791264/Put/seqid=0 2024-11-14T09:16:32,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742383_1559 (size=12151) 2024-11-14T09:16:32,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:32,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731575852775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:32,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:32,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:32,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:32,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:32,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731575852994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:32,997 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_0' in 1d18b03686124f021810b831ea32e069 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:16:32,998 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:32,998 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:32,998 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:33,043 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/d54e098e94ef4e45a1f1eb3e9f5dbe3c 2024-11-14T09:16:33,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/ed08a05d99ca40769f9b6bb4e4d0271f as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/ed08a05d99ca40769f9b6bb4e4d0271f 2024-11-14T09:16:33,050 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/ed08a05d99ca40769f9b6bb4e4d0271f, entries=150, sequenceid=183, filesize=30.4 K 2024-11-14T09:16:33,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/2709a094ac9b461c9aca8d4017c244ce as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/2709a094ac9b461c9aca8d4017c244ce 2024-11-14T09:16:33,054 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/2709a094ac9b461c9aca8d4017c244ce, entries=150, sequenceid=183, filesize=11.9 K 2024-11-14T09:16:33,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/d54e098e94ef4e45a1f1eb3e9f5dbe3c as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/d54e098e94ef4e45a1f1eb3e9f5dbe3c 2024-11-14T09:16:33,057 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/d54e098e94ef4e45a1f1eb3e9f5dbe3c, entries=150, sequenceid=183, filesize=11.9 K 2024-11-14T09:16:33,058 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 1d18b03686124f021810b831ea32e069 in 869ms, sequenceid=183, compaction requested=false 2024-11-14T09:16:33,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 1d18b03686124f021810b831ea32e069: 2024-11-14T09:16:33,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:33,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-14T09:16:33,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-14T09:16:33,060 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-14T09:16:33,060 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9410 sec 2024-11-14T09:16:33,062 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 1.9460 sec 2024-11-14T09:16:33,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on 1d18b03686124f021810b831ea32e069 2024-11-14T09:16:33,087 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1d18b03686124f021810b831ea32e069 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-14T09:16:33,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d18b03686124f021810b831ea32e069, store=A 2024-11-14T09:16:33,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:33,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d18b03686124f021810b831ea32e069, store=B 2024-11-14T09:16:33,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:33,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d18b03686124f021810b831ea32e069, store=C 2024-11-14T09:16:33,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:33,093 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114c1c4ce983c3c41fd942810b77528f1f6_1d18b03686124f021810b831ea32e069 is 50, key is test_row_0/A:col10/1731575793086/Put/seqid=0 2024-11-14T09:16:33,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742384_1560 (size=14794) 2024-11-14T09:16:33,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:33,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731575853157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:33,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:33,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:33,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:33,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-14T09:16:33,265 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:16:33,265 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:16:33,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-14T09:16:33,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-14T09:16:33,267 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:16:33,268 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:16:33,268 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:16:33,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:33,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731575853265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:33,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:33,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:33,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:33,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-14T09:16:33,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-14T09:16:33,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:33,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. as already flushing 2024-11-14T09:16:33,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:33,421 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:33,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:33,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:33,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:33,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731575853474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:33,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:33,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:33,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:33,498 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:33,501 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114c1c4ce983c3c41fd942810b77528f1f6_1d18b03686124f021810b831ea32e069 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c1c4ce983c3c41fd942810b77528f1f6_1d18b03686124f021810b831ea32e069 2024-11-14T09:16:33,502 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/5b0e04409c234d1ca07cb2a08365cf8b, store: [table=TestAcidGuarantees family=A region=1d18b03686124f021810b831ea32e069] 2024-11-14T09:16:33,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/5b0e04409c234d1ca07cb2a08365cf8b is 175, key is test_row_0/A:col10/1731575793086/Put/seqid=0 2024-11-14T09:16:33,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742385_1561 (size=39749) 2024-11-14T09:16:33,574 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-14T09:16:33,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:33,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. as already flushing 2024-11-14T09:16:33,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:33,574 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:33,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:33,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:33,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-14T09:16:33,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:33,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58162 deadline: 1731575853625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:33,628 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_1' in 1d18b03686124f021810b831ea32e069 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:16:33,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:33,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:33,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:33,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:33,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1731575853635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:33,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:33,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58142 deadline: 1731575853635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:33,638 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_0' in 1d18b03686124f021810b831ea32e069 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:16:33,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:33,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:33,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:33,639 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_1' in 1d18b03686124f021810b831ea32e069 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:16:33,639 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:33,639 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:33,639 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:33,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-14T09:16:33,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:33,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. as already flushing 2024-11-14T09:16:33,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:33,727 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:33,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:33,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:33,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:33,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731575853784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:33,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:33,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:33,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:33,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-14T09:16:33,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:33,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. as already flushing 2024-11-14T09:16:33,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:33,880 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:33,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:33,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:33,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-14T09:16:33,906 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=198, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/5b0e04409c234d1ca07cb2a08365cf8b 2024-11-14T09:16:33,913 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/af62a1c0ac4a491bb8a2ed00a1ff14d0 is 50, key is test_row_0/B:col10/1731575793086/Put/seqid=0 2024-11-14T09:16:33,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742386_1562 (size=12151) 2024-11-14T09:16:34,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-14T09:16:34,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:34,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. as already flushing 2024-11-14T09:16:34,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:34,033 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:34,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:34,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:34,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-14T09:16:34,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:34,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. as already flushing 2024-11-14T09:16:34,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:34,185 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:34,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:34,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:34,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:34,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731575854295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:34,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:34,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:34,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:34,317 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/af62a1c0ac4a491bb8a2ed00a1ff14d0 2024-11-14T09:16:34,325 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/f7e065fac6364702a452ef75d6dc0121 is 50, key is test_row_0/C:col10/1731575793086/Put/seqid=0 2024-11-14T09:16:34,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742387_1563 (size=12151) 2024-11-14T09:16:34,338 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-14T09:16:34,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:34,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. as already flushing 2024-11-14T09:16:34,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:34,338 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:34,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:34,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:34,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-14T09:16:34,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-14T09:16:34,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:34,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. as already flushing 2024-11-14T09:16:34,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:34,491 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:34,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:34,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:34,643 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-14T09:16:34,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:34,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. as already flushing 2024-11-14T09:16:34,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:34,644 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:34,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:34,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:34,729 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/f7e065fac6364702a452ef75d6dc0121 2024-11-14T09:16:34,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/5b0e04409c234d1ca07cb2a08365cf8b as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/5b0e04409c234d1ca07cb2a08365cf8b 2024-11-14T09:16:34,736 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/5b0e04409c234d1ca07cb2a08365cf8b, entries=200, sequenceid=198, filesize=38.8 K 2024-11-14T09:16:34,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/af62a1c0ac4a491bb8a2ed00a1ff14d0 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/af62a1c0ac4a491bb8a2ed00a1ff14d0 2024-11-14T09:16:34,739 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/af62a1c0ac4a491bb8a2ed00a1ff14d0, entries=150, sequenceid=198, filesize=11.9 K 2024-11-14T09:16:34,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/f7e065fac6364702a452ef75d6dc0121 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/f7e065fac6364702a452ef75d6dc0121 2024-11-14T09:16:34,742 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/f7e065fac6364702a452ef75d6dc0121, entries=150, sequenceid=198, filesize=11.9 K 2024-11-14T09:16:34,743 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 1d18b03686124f021810b831ea32e069 in 1656ms, sequenceid=198, compaction requested=true 2024-11-14T09:16:34,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1d18b03686124f021810b831ea32e069: 2024-11-14T09:16:34,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d18b03686124f021810b831ea32e069:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:16:34,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:16:34,744 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:16:34,744 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:16:34,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d18b03686124f021810b831ea32e069:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:16:34,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:16:34,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d18b03686124f021810b831ea32e069:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:16:34,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:16:34,745 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102267 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:16:34,745 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:16:34,745 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): 1d18b03686124f021810b831ea32e069/A is initiating minor compaction (all files) 2024-11-14T09:16:34,745 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): 1d18b03686124f021810b831ea32e069/B is initiating minor compaction (all files) 2024-11-14T09:16:34,745 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d18b03686124f021810b831ea32e069/A in TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:34,745 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d18b03686124f021810b831ea32e069/B in TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:34,745 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/c12bb0e24fa24ca988d909d9d5b457f2, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/ed08a05d99ca40769f9b6bb4e4d0271f, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/5b0e04409c234d1ca07cb2a08365cf8b] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp, totalSize=99.9 K 2024-11-14T09:16:34,745 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/b00111888185487c94bbd140cf7406a7, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/2709a094ac9b461c9aca8d4017c244ce, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/af62a1c0ac4a491bb8a2ed00a1ff14d0] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp, totalSize=35.9 K 2024-11-14T09:16:34,745 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:34,745 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. files: [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/c12bb0e24fa24ca988d909d9d5b457f2, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/ed08a05d99ca40769f9b6bb4e4d0271f, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/5b0e04409c234d1ca07cb2a08365cf8b] 2024-11-14T09:16:34,745 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting b00111888185487c94bbd140cf7406a7, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1731575789445 2024-11-14T09:16:34,746 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting c12bb0e24fa24ca988d909d9d5b457f2, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1731575789445 2024-11-14T09:16:34,746 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 2709a094ac9b461c9aca8d4017c244ce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731575791244 2024-11-14T09:16:34,746 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed08a05d99ca40769f9b6bb4e4d0271f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731575791244 2024-11-14T09:16:34,746 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting af62a1c0ac4a491bb8a2ed00a1ff14d0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731575792442 2024-11-14T09:16:34,746 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b0e04409c234d1ca07cb2a08365cf8b, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731575792442 2024-11-14T09:16:34,751 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1d18b03686124f021810b831ea32e069] 2024-11-14T09:16:34,753 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d18b03686124f021810b831ea32e069#B#compaction#477 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:16:34,753 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/ba3198e507eb4693a2bbde1bf3ff4a93 is 50, key is test_row_0/B:col10/1731575793086/Put/seqid=0 2024-11-14T09:16:34,754 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114c6437b1beaf64292b652f031fd0403d3_1d18b03686124f021810b831ea32e069 store=[table=TestAcidGuarantees family=A region=1d18b03686124f021810b831ea32e069] 2024-11-14T09:16:34,756 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114c6437b1beaf64292b652f031fd0403d3_1d18b03686124f021810b831ea32e069, store=[table=TestAcidGuarantees family=A region=1d18b03686124f021810b831ea32e069] 2024-11-14T09:16:34,756 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114c6437b1beaf64292b652f031fd0403d3_1d18b03686124f021810b831ea32e069 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1d18b03686124f021810b831ea32e069] 2024-11-14T09:16:34,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742389_1565 (size=4469) 2024-11-14T09:16:34,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742388_1564 (size=12561) 2024-11-14T09:16:34,796 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-14T09:16:34,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:34,796 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 1d18b03686124f021810b831ea32e069 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-14T09:16:34,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d18b03686124f021810b831ea32e069, store=A 2024-11-14T09:16:34,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:34,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d18b03686124f021810b831ea32e069, store=B 2024-11-14T09:16:34,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:34,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d18b03686124f021810b831ea32e069, store=C 2024-11-14T09:16:34,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:34,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114dfa3bee8126b4c59bdd4fd4d73fccdbd_1d18b03686124f021810b831ea32e069 is 50, key is test_row_0/A:col10/1731575793150/Put/seqid=0 2024-11-14T09:16:34,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742390_1566 (size=12304) 2024-11-14T09:16:35,164 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d18b03686124f021810b831ea32e069#A#compaction#478 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:16:35,164 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/dd4aada6a1944c28b77b4d71ee5c5743 is 175, key is test_row_0/A:col10/1731575793086/Put/seqid=0 2024-11-14T09:16:35,169 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/ba3198e507eb4693a2bbde1bf3ff4a93 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/ba3198e507eb4693a2bbde1bf3ff4a93 2024-11-14T09:16:35,173 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d18b03686124f021810b831ea32e069/B of 1d18b03686124f021810b831ea32e069 into ba3198e507eb4693a2bbde1bf3ff4a93(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:16:35,173 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d18b03686124f021810b831ea32e069: 2024-11-14T09:16:35,173 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., storeName=1d18b03686124f021810b831ea32e069/B, priority=13, startTime=1731575794744; duration=0sec 2024-11-14T09:16:35,173 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:16:35,173 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d18b03686124f021810b831ea32e069:B 2024-11-14T09:16:35,173 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:16:35,174 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:16:35,174 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): 1d18b03686124f021810b831ea32e069/C is initiating minor compaction (all files) 2024-11-14T09:16:35,174 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d18b03686124f021810b831ea32e069/C in TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:35,174 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/54b3756177fd4d8281dc31d657ef607d, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/d54e098e94ef4e45a1f1eb3e9f5dbe3c, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/f7e065fac6364702a452ef75d6dc0121] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp, totalSize=35.9 K 2024-11-14T09:16:35,175 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 54b3756177fd4d8281dc31d657ef607d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1731575789445 2024-11-14T09:16:35,175 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting d54e098e94ef4e45a1f1eb3e9f5dbe3c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731575791244 2024-11-14T09:16:35,175 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting f7e065fac6364702a452ef75d6dc0121, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731575792442 2024-11-14T09:16:35,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742391_1567 (size=31515) 2024-11-14T09:16:35,191 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d18b03686124f021810b831ea32e069#C#compaction#480 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:16:35,192 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/0e2fd4c37d104ae693123f54b2437055 is 50, key is test_row_0/C:col10/1731575793086/Put/seqid=0 2024-11-14T09:16:35,195 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/dd4aada6a1944c28b77b4d71ee5c5743 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/dd4aada6a1944c28b77b4d71ee5c5743 2024-11-14T09:16:35,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742392_1568 (size=12561) 2024-11-14T09:16:35,200 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d18b03686124f021810b831ea32e069/A of 1d18b03686124f021810b831ea32e069 into dd4aada6a1944c28b77b4d71ee5c5743(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:16:35,200 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d18b03686124f021810b831ea32e069: 2024-11-14T09:16:35,200 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., storeName=1d18b03686124f021810b831ea32e069/A, priority=13, startTime=1731575794743; duration=0sec 2024-11-14T09:16:35,200 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:16:35,200 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d18b03686124f021810b831ea32e069:A 2024-11-14T09:16:35,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:35,214 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114dfa3bee8126b4c59bdd4fd4d73fccdbd_1d18b03686124f021810b831ea32e069 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114dfa3bee8126b4c59bdd4fd4d73fccdbd_1d18b03686124f021810b831ea32e069 2024-11-14T09:16:35,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/96336ee5892a49e3a482d68db457fc5e, store: [table=TestAcidGuarantees family=A region=1d18b03686124f021810b831ea32e069] 2024-11-14T09:16:35,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/96336ee5892a49e3a482d68db457fc5e is 175, key is test_row_0/A:col10/1731575793150/Put/seqid=0 2024-11-14T09:16:35,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742393_1569 (size=31105) 2024-11-14T09:16:35,219 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=216, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/96336ee5892a49e3a482d68db457fc5e 2024-11-14T09:16:35,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/c24c5a3f8fcc49f98a109563330f6a5e is 50, key is test_row_0/B:col10/1731575793150/Put/seqid=0 2024-11-14T09:16:35,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742394_1570 (size=12151) 2024-11-14T09:16:35,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on 1d18b03686124f021810b831ea32e069 2024-11-14T09:16:35,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. as already flushing 2024-11-14T09:16:35,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:35,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731575855350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:35,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:35,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:35,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:35,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-14T09:16:35,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:35,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731575855465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:35,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:35,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:35,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:35,601 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/0e2fd4c37d104ae693123f54b2437055 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/0e2fd4c37d104ae693123f54b2437055 2024-11-14T09:16:35,605 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d18b03686124f021810b831ea32e069/C of 1d18b03686124f021810b831ea32e069 into 0e2fd4c37d104ae693123f54b2437055(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:16:35,605 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d18b03686124f021810b831ea32e069: 2024-11-14T09:16:35,605 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., storeName=1d18b03686124f021810b831ea32e069/C, priority=13, startTime=1731575794744; duration=0sec 2024-11-14T09:16:35,605 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:16:35,605 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d18b03686124f021810b831ea32e069:C 2024-11-14T09:16:35,629 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/c24c5a3f8fcc49f98a109563330f6a5e 2024-11-14T09:16:35,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/ff79faf6140a421cb4796ac25a9faacc is 50, key is test_row_0/C:col10/1731575793150/Put/seqid=0 2024-11-14T09:16:35,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742395_1571 (size=12151) 2024-11-14T09:16:35,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:35,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731575855675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:35,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:35,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:35,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:35,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:35,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731575855984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:35,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:35,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:35,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:36,041 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/ff79faf6140a421cb4796ac25a9faacc 2024-11-14T09:16:36,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/96336ee5892a49e3a482d68db457fc5e as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/96336ee5892a49e3a482d68db457fc5e 2024-11-14T09:16:36,049 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/96336ee5892a49e3a482d68db457fc5e, entries=150, sequenceid=216, filesize=30.4 K 2024-11-14T09:16:36,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/c24c5a3f8fcc49f98a109563330f6a5e as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/c24c5a3f8fcc49f98a109563330f6a5e 2024-11-14T09:16:36,060 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/c24c5a3f8fcc49f98a109563330f6a5e, entries=150, sequenceid=216, filesize=11.9 K 2024-11-14T09:16:36,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/ff79faf6140a421cb4796ac25a9faacc as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/ff79faf6140a421cb4796ac25a9faacc 2024-11-14T09:16:36,063 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/ff79faf6140a421cb4796ac25a9faacc, entries=150, sequenceid=216, filesize=11.9 K 2024-11-14T09:16:36,064 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 1d18b03686124f021810b831ea32e069 in 1268ms, sequenceid=216, compaction requested=false 2024-11-14T09:16:36,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 1d18b03686124f021810b831ea32e069: 2024-11-14T09:16:36,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:36,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-14T09:16:36,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-14T09:16:36,068 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-14T09:16:36,068 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7980 sec 2024-11-14T09:16:36,069 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 2.8030 sec 2024-11-14T09:16:36,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on 1d18b03686124f021810b831ea32e069 2024-11-14T09:16:36,499 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1d18b03686124f021810b831ea32e069 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-14T09:16:36,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d18b03686124f021810b831ea32e069, store=A 2024-11-14T09:16:36,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:36,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d18b03686124f021810b831ea32e069, store=B 2024-11-14T09:16:36,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:36,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d18b03686124f021810b831ea32e069, store=C 2024-11-14T09:16:36,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:36,505 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114ae2c7d0053334823b30bc32bbb2607aa_1d18b03686124f021810b831ea32e069 is 50, key is test_row_0/A:col10/1731575796497/Put/seqid=0 2024-11-14T09:16:36,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742396_1572 (size=14794) 2024-11-14T09:16:36,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:36,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731575856564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:36,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:36,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:36,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:36,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:36,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731575856685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:36,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:36,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:36,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:36,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:36,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731575856894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:36,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:36,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:36,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:36,909 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:36,912 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114ae2c7d0053334823b30bc32bbb2607aa_1d18b03686124f021810b831ea32e069 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ae2c7d0053334823b30bc32bbb2607aa_1d18b03686124f021810b831ea32e069 2024-11-14T09:16:36,913 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/2f33cc2520c047979b34a7c9a752c1eb, store: [table=TestAcidGuarantees family=A region=1d18b03686124f021810b831ea32e069] 2024-11-14T09:16:36,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/2f33cc2520c047979b34a7c9a752c1eb is 175, key is test_row_0/A:col10/1731575796497/Put/seqid=0 2024-11-14T09:16:36,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742397_1573 (size=39749) 2024-11-14T09:16:37,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:37,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731575857025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:37,029 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_0' in 1d18b03686124f021810b831ea32e069 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:16:37,030 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:37,030 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:37,030 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:37,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:37,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731575857205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:37,207 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:37,207 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:37,207 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:37,318 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=234, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/2f33cc2520c047979b34a7c9a752c1eb 2024-11-14T09:16:37,334 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/69e91c71adf64f3982864a845b629369 is 50, key is test_row_0/B:col10/1731575796497/Put/seqid=0 2024-11-14T09:16:37,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742398_1574 (size=12151) 2024-11-14T09:16:37,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-14T09:16:37,436 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:16:37,437 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:16:37,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-14T09:16:37,440 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:16:37,441 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:16:37,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-14T09:16:37,441 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:16:37,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-14T09:16:37,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-14T09:16:37,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:37,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. as already flushing 2024-11-14T09:16:37,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:37,594 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:37,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:37,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:37,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:37,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58162 deadline: 1731575857645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:37,647 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_1' in 1d18b03686124f021810b831ea32e069 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:16:37,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:37,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:37,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:37,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:37,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1731575857675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:37,680 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_0' in 1d18b03686124f021810b831ea32e069 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8231 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:16:37,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:37,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:37,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:37,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:37,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58142 deadline: 1731575857684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:37,686 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 83f56b55f2af,39927,1731575622393 for 'test_row_1' in 1d18b03686124f021810b831ea32e069 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8236 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T09:16:37,687 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:37,687 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:37,687 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:37,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:37,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731575857715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:37,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:37,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:37,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:37,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-14T09:16:37,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:37,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. as already flushing 2024-11-14T09:16:37,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:37,747 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:37,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:37,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:37,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-14T09:16:37,770 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/69e91c71adf64f3982864a845b629369 2024-11-14T09:16:37,777 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/420b9f987921497e979e00b0e68eb8e4 is 50, key is test_row_0/C:col10/1731575796497/Put/seqid=0 2024-11-14T09:16:37,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742399_1575 (size=12151) 2024-11-14T09:16:37,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-14T09:16:37,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:37,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. as already flushing 2024-11-14T09:16:37,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:37,899 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:37,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:37,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:38,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-14T09:16:38,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:38,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. as already flushing 2024-11-14T09:16:38,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:38,052 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:38,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:38,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:38,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-14T09:16:38,181 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/420b9f987921497e979e00b0e68eb8e4 2024-11-14T09:16:38,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/2f33cc2520c047979b34a7c9a752c1eb as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/2f33cc2520c047979b34a7c9a752c1eb 2024-11-14T09:16:38,188 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/2f33cc2520c047979b34a7c9a752c1eb, entries=200, sequenceid=234, filesize=38.8 K 2024-11-14T09:16:38,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/69e91c71adf64f3982864a845b629369 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/69e91c71adf64f3982864a845b629369 2024-11-14T09:16:38,191 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/69e91c71adf64f3982864a845b629369, entries=150, sequenceid=234, filesize=11.9 K 2024-11-14T09:16:38,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/420b9f987921497e979e00b0e68eb8e4 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/420b9f987921497e979e00b0e68eb8e4 2024-11-14T09:16:38,195 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/420b9f987921497e979e00b0e68eb8e4, entries=150, sequenceid=234, filesize=11.9 K 2024-11-14T09:16:38,196 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 1d18b03686124f021810b831ea32e069 in 1697ms, sequenceid=234, compaction requested=true 2024-11-14T09:16:38,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1d18b03686124f021810b831ea32e069: 2024-11-14T09:16:38,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d18b03686124f021810b831ea32e069:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:16:38,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:16:38,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d18b03686124f021810b831ea32e069:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:16:38,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:16:38,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d18b03686124f021810b831ea32e069:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:16:38,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:16:38,196 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:16:38,196 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:16:38,197 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:16:38,197 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102369 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:16:38,197 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): 1d18b03686124f021810b831ea32e069/B is initiating minor compaction (all files) 2024-11-14T09:16:38,197 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): 1d18b03686124f021810b831ea32e069/A is initiating minor compaction (all files) 2024-11-14T09:16:38,197 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d18b03686124f021810b831ea32e069/B in TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:38,197 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d18b03686124f021810b831ea32e069/A in TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:38,197 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/ba3198e507eb4693a2bbde1bf3ff4a93, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/c24c5a3f8fcc49f98a109563330f6a5e, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/69e91c71adf64f3982864a845b629369] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp, totalSize=36.0 K 2024-11-14T09:16:38,197 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/dd4aada6a1944c28b77b4d71ee5c5743, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/96336ee5892a49e3a482d68db457fc5e, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/2f33cc2520c047979b34a7c9a752c1eb] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp, totalSize=100.0 K 2024-11-14T09:16:38,197 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:38,197 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. files: [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/dd4aada6a1944c28b77b4d71ee5c5743, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/96336ee5892a49e3a482d68db457fc5e, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/2f33cc2520c047979b34a7c9a752c1eb] 2024-11-14T09:16:38,197 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting ba3198e507eb4693a2bbde1bf3ff4a93, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731575792442 2024-11-14T09:16:38,197 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd4aada6a1944c28b77b4d71ee5c5743, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731575792442 2024-11-14T09:16:38,197 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting c24c5a3f8fcc49f98a109563330f6a5e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731575793144 2024-11-14T09:16:38,197 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96336ee5892a49e3a482d68db457fc5e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731575793144 2024-11-14T09:16:38,198 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 69e91c71adf64f3982864a845b629369, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731575795345 2024-11-14T09:16:38,198 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f33cc2520c047979b34a7c9a752c1eb, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731575795341 2024-11-14T09:16:38,202 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1d18b03686124f021810b831ea32e069] 2024-11-14T09:16:38,203 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d18b03686124f021810b831ea32e069#B#compaction#486 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:16:38,204 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/15271176d80d4702ae1077a7d44726ad is 50, key is test_row_0/B:col10/1731575796497/Put/seqid=0 2024-11-14T09:16:38,205 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411148cee2bfceea44c5d9d8ac0737e939850_1d18b03686124f021810b831ea32e069 store=[table=TestAcidGuarantees family=A region=1d18b03686124f021810b831ea32e069] 2024-11-14T09:16:38,205 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-14T09:16:38,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:38,206 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 1d18b03686124f021810b831ea32e069 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-14T09:16:38,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d18b03686124f021810b831ea32e069, store=A 2024-11-14T09:16:38,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:38,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d18b03686124f021810b831ea32e069, store=B 2024-11-14T09:16:38,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:38,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d18b03686124f021810b831ea32e069, store=C 2024-11-14T09:16:38,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:38,206 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411148cee2bfceea44c5d9d8ac0737e939850_1d18b03686124f021810b831ea32e069, store=[table=TestAcidGuarantees family=A region=1d18b03686124f021810b831ea32e069] 2024-11-14T09:16:38,206 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411148cee2bfceea44c5d9d8ac0737e939850_1d18b03686124f021810b831ea32e069 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1d18b03686124f021810b831ea32e069] 2024-11-14T09:16:38,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742400_1576 (size=12663) 2024-11-14T09:16:38,214 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/15271176d80d4702ae1077a7d44726ad as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/15271176d80d4702ae1077a7d44726ad 2024-11-14T09:16:38,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742401_1577 (size=4469) 2024-11-14T09:16:38,218 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d18b03686124f021810b831ea32e069/B of 1d18b03686124f021810b831ea32e069 into 15271176d80d4702ae1077a7d44726ad(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:16:38,218 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d18b03686124f021810b831ea32e069: 2024-11-14T09:16:38,218 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., storeName=1d18b03686124f021810b831ea32e069/B, priority=13, startTime=1731575798196; duration=0sec 2024-11-14T09:16:38,218 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:16:38,218 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d18b03686124f021810b831ea32e069:B 2024-11-14T09:16:38,218 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:16:38,219 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:16:38,219 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): 1d18b03686124f021810b831ea32e069/C is initiating minor compaction (all files) 2024-11-14T09:16:38,219 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d18b03686124f021810b831ea32e069/C in TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:38,219 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/0e2fd4c37d104ae693123f54b2437055, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/ff79faf6140a421cb4796ac25a9faacc, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/420b9f987921497e979e00b0e68eb8e4] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp, totalSize=36.0 K 2024-11-14T09:16:38,220 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e2fd4c37d104ae693123f54b2437055, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731575792442 2024-11-14T09:16:38,220 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting ff79faf6140a421cb4796ac25a9faacc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731575793144 2024-11-14T09:16:38,220 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 420b9f987921497e979e00b0e68eb8e4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731575795345 2024-11-14T09:16:38,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411147bd4882f649147e6bd0b8a60fbcd9ed1_1d18b03686124f021810b831ea32e069 is 50, key is test_row_0/A:col10/1731575796562/Put/seqid=0 2024-11-14T09:16:38,227 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d18b03686124f021810b831ea32e069#C#compaction#489 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:16:38,227 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/7761a02d17c64f6d874518fb4c179568 is 50, key is test_row_0/C:col10/1731575796497/Put/seqid=0 2024-11-14T09:16:38,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742402_1578 (size=12304) 2024-11-14T09:16:38,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742403_1579 (size=12663) 2024-11-14T09:16:38,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-14T09:16:38,619 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d18b03686124f021810b831ea32e069#A#compaction#487 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:16:38,620 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/3b74aebb23684c8abbc886bf240a8e10 is 175, key is test_row_0/A:col10/1731575796497/Put/seqid=0 2024-11-14T09:16:38,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742404_1580 (size=31617) 2024-11-14T09:16:38,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:38,633 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411147bd4882f649147e6bd0b8a60fbcd9ed1_1d18b03686124f021810b831ea32e069 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411147bd4882f649147e6bd0b8a60fbcd9ed1_1d18b03686124f021810b831ea32e069 2024-11-14T09:16:38,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/09a1ccd087574c268c8bf8c6c326a798, store: [table=TestAcidGuarantees family=A region=1d18b03686124f021810b831ea32e069] 2024-11-14T09:16:38,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/09a1ccd087574c268c8bf8c6c326a798 is 175, key is test_row_0/A:col10/1731575796562/Put/seqid=0 2024-11-14T09:16:38,637 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/7761a02d17c64f6d874518fb4c179568 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/7761a02d17c64f6d874518fb4c179568 2024-11-14T09:16:38,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742405_1581 (size=31105) 2024-11-14T09:16:38,642 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d18b03686124f021810b831ea32e069/C of 1d18b03686124f021810b831ea32e069 into 7761a02d17c64f6d874518fb4c179568(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:16:38,642 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d18b03686124f021810b831ea32e069: 2024-11-14T09:16:38,642 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., storeName=1d18b03686124f021810b831ea32e069/C, priority=13, startTime=1731575798196; duration=0sec 2024-11-14T09:16:38,642 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:16:38,642 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d18b03686124f021810b831ea32e069:C 2024-11-14T09:16:38,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on 1d18b03686124f021810b831ea32e069 2024-11-14T09:16:38,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. as already flushing 2024-11-14T09:16:38,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:38,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731575858785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:38,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:38,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:38,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:38,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:38,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731575858895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:38,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:38,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:38,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:39,027 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/3b74aebb23684c8abbc886bf240a8e10 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/3b74aebb23684c8abbc886bf240a8e10 2024-11-14T09:16:39,032 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d18b03686124f021810b831ea32e069/A of 1d18b03686124f021810b831ea32e069 into 3b74aebb23684c8abbc886bf240a8e10(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:16:39,032 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d18b03686124f021810b831ea32e069: 2024-11-14T09:16:39,032 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., storeName=1d18b03686124f021810b831ea32e069/A, priority=13, startTime=1731575798196; duration=0sec 2024-11-14T09:16:39,032 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:16:39,032 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d18b03686124f021810b831ea32e069:A 2024-11-14T09:16:39,039 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=251, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/09a1ccd087574c268c8bf8c6c326a798 2024-11-14T09:16:39,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/361ac23c0e094cb3a7325693742c02f5 is 50, key is test_row_0/B:col10/1731575796562/Put/seqid=0 2024-11-14T09:16:39,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742406_1582 (size=12151) 2024-11-14T09:16:39,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:39,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731575859104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:39,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:39,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:39,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:39,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:39,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731575859415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:39,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:39,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:39,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:39,450 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/361ac23c0e094cb3a7325693742c02f5 2024-11-14T09:16:39,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/29328a346cf949d2aa4d90a1cd50506c is 50, key is test_row_0/C:col10/1731575796562/Put/seqid=0 2024-11-14T09:16:39,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742407_1583 (size=12151) 2024-11-14T09:16:39,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-14T09:16:39,866 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/29328a346cf949d2aa4d90a1cd50506c 2024-11-14T09:16:39,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/09a1ccd087574c268c8bf8c6c326a798 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/09a1ccd087574c268c8bf8c6c326a798 2024-11-14T09:16:39,873 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/09a1ccd087574c268c8bf8c6c326a798, entries=150, sequenceid=251, filesize=30.4 K 2024-11-14T09:16:39,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/361ac23c0e094cb3a7325693742c02f5 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/361ac23c0e094cb3a7325693742c02f5 2024-11-14T09:16:39,877 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/361ac23c0e094cb3a7325693742c02f5, entries=150, sequenceid=251, filesize=11.9 K 2024-11-14T09:16:39,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/29328a346cf949d2aa4d90a1cd50506c as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/29328a346cf949d2aa4d90a1cd50506c 2024-11-14T09:16:39,880 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/29328a346cf949d2aa4d90a1cd50506c, entries=150, sequenceid=251, filesize=11.9 K 2024-11-14T09:16:39,881 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 1d18b03686124f021810b831ea32e069 in 1676ms, sequenceid=251, compaction requested=false 2024-11-14T09:16:39,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 1d18b03686124f021810b831ea32e069: 2024-11-14T09:16:39,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:39,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-14T09:16:39,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-14T09:16:39,884 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-14T09:16:39,884 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4410 sec 2024-11-14T09:16:39,885 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 2.4470 sec 2024-11-14T09:16:39,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on 1d18b03686124f021810b831ea32e069 2024-11-14T09:16:39,928 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1d18b03686124f021810b831ea32e069 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-14T09:16:39,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d18b03686124f021810b831ea32e069, store=A 2024-11-14T09:16:39,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:39,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d18b03686124f021810b831ea32e069, store=B 2024-11-14T09:16:39,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:39,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d18b03686124f021810b831ea32e069, store=C 2024-11-14T09:16:39,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:39,935 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411142aaef23e5c30431f872232d38cd592d5_1d18b03686124f021810b831ea32e069 is 50, key is test_row_0/A:col10/1731575798783/Put/seqid=0 2024-11-14T09:16:39,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742408_1584 (size=14994) 2024-11-14T09:16:39,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:39,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731575859983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:39,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:39,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:39,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:40,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:40,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731575860095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:40,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:40,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:40,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:40,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:40,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731575860304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:40,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:40,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:40,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:40,339 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:40,342 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411142aaef23e5c30431f872232d38cd592d5_1d18b03686124f021810b831ea32e069 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411142aaef23e5c30431f872232d38cd592d5_1d18b03686124f021810b831ea32e069 2024-11-14T09:16:40,342 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/c0eb6508740c47ba84320e5e096577e5, store: [table=TestAcidGuarantees family=A region=1d18b03686124f021810b831ea32e069] 2024-11-14T09:16:40,343 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/c0eb6508740c47ba84320e5e096577e5 is 175, key is test_row_0/A:col10/1731575798783/Put/seqid=0 2024-11-14T09:16:40,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742409_1585 (size=39949) 2024-11-14T09:16:40,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:40,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731575860615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:40,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:40,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:40,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:40,748 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=270, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/c0eb6508740c47ba84320e5e096577e5 2024-11-14T09:16:40,754 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T09:16:40,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/b889837572fb40f7b8d5f72c1f7c39a7 is 50, key is test_row_0/B:col10/1731575798783/Put/seqid=0 2024-11-14T09:16:40,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742410_1586 (size=12301) 2024-11-14T09:16:41,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:41,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731575861124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:41,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:41,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:41,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:41,159 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/b889837572fb40f7b8d5f72c1f7c39a7 2024-11-14T09:16:41,166 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/624e5540b0214f348a422fc18aa6afc8 is 50, key is test_row_0/C:col10/1731575798783/Put/seqid=0 2024-11-14T09:16:41,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742411_1587 (size=12301) 2024-11-14T09:16:41,569 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/624e5540b0214f348a422fc18aa6afc8 2024-11-14T09:16:41,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/c0eb6508740c47ba84320e5e096577e5 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/c0eb6508740c47ba84320e5e096577e5 2024-11-14T09:16:41,576 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/c0eb6508740c47ba84320e5e096577e5, entries=200, sequenceid=270, filesize=39.0 K 2024-11-14T09:16:41,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/b889837572fb40f7b8d5f72c1f7c39a7 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/b889837572fb40f7b8d5f72c1f7c39a7 2024-11-14T09:16:41,580 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/b889837572fb40f7b8d5f72c1f7c39a7, entries=150, sequenceid=270, filesize=12.0 K 2024-11-14T09:16:41,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/624e5540b0214f348a422fc18aa6afc8 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/624e5540b0214f348a422fc18aa6afc8 2024-11-14T09:16:41,584 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/624e5540b0214f348a422fc18aa6afc8, entries=150, sequenceid=270, filesize=12.0 K 2024-11-14T09:16:41,585 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 1d18b03686124f021810b831ea32e069 in 1658ms, sequenceid=270, compaction requested=true 2024-11-14T09:16:41,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1d18b03686124f021810b831ea32e069: 2024-11-14T09:16:41,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d18b03686124f021810b831ea32e069:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:16:41,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:16:41,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d18b03686124f021810b831ea32e069:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:16:41,585 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:16:41,585 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:16:41,586 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102671 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:16:41,586 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): 1d18b03686124f021810b831ea32e069/A is initiating minor compaction (all files) 2024-11-14T09:16:41,586 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:16:41,586 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d18b03686124f021810b831ea32e069/A in TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:41,586 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): 1d18b03686124f021810b831ea32e069/B is initiating minor compaction (all files) 2024-11-14T09:16:41,586 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/3b74aebb23684c8abbc886bf240a8e10, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/09a1ccd087574c268c8bf8c6c326a798, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/c0eb6508740c47ba84320e5e096577e5] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp, totalSize=100.3 K 2024-11-14T09:16:41,586 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:41,586 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d18b03686124f021810b831ea32e069/B in TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:41,586 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. files: [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/3b74aebb23684c8abbc886bf240a8e10, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/09a1ccd087574c268c8bf8c6c326a798, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/c0eb6508740c47ba84320e5e096577e5] 2024-11-14T09:16:41,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:16:41,586 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/15271176d80d4702ae1077a7d44726ad, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/361ac23c0e094cb3a7325693742c02f5, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/b889837572fb40f7b8d5f72c1f7c39a7] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp, totalSize=36.2 K 2024-11-14T09:16:41,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d18b03686124f021810b831ea32e069:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:16:41,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:16:41,586 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b74aebb23684c8abbc886bf240a8e10, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731575795345 2024-11-14T09:16:41,586 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15271176d80d4702ae1077a7d44726ad, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731575795345 2024-11-14T09:16:41,587 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 09a1ccd087574c268c8bf8c6c326a798, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731575796541 2024-11-14T09:16:41,587 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 361ac23c0e094cb3a7325693742c02f5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731575796541 2024-11-14T09:16:41,587 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting c0eb6508740c47ba84320e5e096577e5, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731575798746 2024-11-14T09:16:41,587 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting b889837572fb40f7b8d5f72c1f7c39a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731575798746 2024-11-14T09:16:41,592 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1d18b03686124f021810b831ea32e069] 2024-11-14T09:16:41,593 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d18b03686124f021810b831ea32e069#B#compaction#496 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:16:41,593 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/0fc23249934744688eca69a8a12c9a25 is 50, key is test_row_0/B:col10/1731575798783/Put/seqid=0 2024-11-14T09:16:41,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-14T09:16:41,595 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:16:41,596 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:16:41,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-14T09:16:41,597 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114a34bbe06946341b2af98782bc46979f8_1d18b03686124f021810b831ea32e069 store=[table=TestAcidGuarantees family=A region=1d18b03686124f021810b831ea32e069] 2024-11-14T09:16:41,597 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:16:41,598 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:16:41,598 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:16:41,599 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114a34bbe06946341b2af98782bc46979f8_1d18b03686124f021810b831ea32e069, store=[table=TestAcidGuarantees family=A region=1d18b03686124f021810b831ea32e069] 2024-11-14T09:16:41,599 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114a34bbe06946341b2af98782bc46979f8_1d18b03686124f021810b831ea32e069 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1d18b03686124f021810b831ea32e069] 2024-11-14T09:16:41,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-14T09:16:41,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742412_1588 (size=12915) 2024-11-14T09:16:41,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742413_1589 (size=4469) 2024-11-14T09:16:41,611 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d18b03686124f021810b831ea32e069#A#compaction#495 average throughput is 1.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:16:41,612 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/74c355a3333d48e9b6b50f0e75a5a21b is 175, key is test_row_0/A:col10/1731575798783/Put/seqid=0 2024-11-14T09:16:41,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742414_1590 (size=31869) 2024-11-14T09:16:41,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-14T09:16:41,752 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-14T09:16:41,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:41,752 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 1d18b03686124f021810b831ea32e069 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-14T09:16:41,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d18b03686124f021810b831ea32e069, store=A 2024-11-14T09:16:41,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:41,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d18b03686124f021810b831ea32e069, store=B 2024-11-14T09:16:41,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:41,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d18b03686124f021810b831ea32e069, store=C 2024-11-14T09:16:41,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:41,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111485d174878d1741ccb57536ffe78d036a_1d18b03686124f021810b831ea32e069 is 50, key is test_row_0/A:col10/1731575799967/Put/seqid=0 2024-11-14T09:16:41,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742415_1591 (size=12454) 2024-11-14T09:16:41,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:41,766 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111485d174878d1741ccb57536ffe78d036a_1d18b03686124f021810b831ea32e069 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111485d174878d1741ccb57536ffe78d036a_1d18b03686124f021810b831ea32e069 2024-11-14T09:16:41,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/d8215dfd9f01413191955c5ab715d42c, store: [table=TestAcidGuarantees family=A region=1d18b03686124f021810b831ea32e069] 2024-11-14T09:16:41,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/d8215dfd9f01413191955c5ab715d42c is 175, key is test_row_0/A:col10/1731575799967/Put/seqid=0 2024-11-14T09:16:41,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742416_1592 (size=31255) 2024-11-14T09:16:41,772 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=286, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/d8215dfd9f01413191955c5ab715d42c 2024-11-14T09:16:41,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/952d378d76f84d17b56bc68a21326a62 is 50, key is test_row_0/B:col10/1731575799967/Put/seqid=0 2024-11-14T09:16:41,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742417_1593 (size=12301) 2024-11-14T09:16:41,781 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/952d378d76f84d17b56bc68a21326a62 2024-11-14T09:16:41,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/6c10e8a0b3d840da90aca0d2b977f664 is 50, key is test_row_0/C:col10/1731575799967/Put/seqid=0 2024-11-14T09:16:41,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742418_1594 (size=12301) 2024-11-14T09:16:41,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-14T09:16:42,010 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/0fc23249934744688eca69a8a12c9a25 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/0fc23249934744688eca69a8a12c9a25 2024-11-14T09:16:42,014 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d18b03686124f021810b831ea32e069/B of 1d18b03686124f021810b831ea32e069 into 0fc23249934744688eca69a8a12c9a25(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:16:42,014 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d18b03686124f021810b831ea32e069: 2024-11-14T09:16:42,014 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., storeName=1d18b03686124f021810b831ea32e069/B, priority=13, startTime=1731575801585; duration=0sec 2024-11-14T09:16:42,014 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:16:42,014 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d18b03686124f021810b831ea32e069:B 2024-11-14T09:16:42,014 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:16:42,015 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:16:42,015 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): 1d18b03686124f021810b831ea32e069/C is initiating minor compaction (all files) 2024-11-14T09:16:42,015 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d18b03686124f021810b831ea32e069/C in TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:42,015 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/7761a02d17c64f6d874518fb4c179568, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/29328a346cf949d2aa4d90a1cd50506c, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/624e5540b0214f348a422fc18aa6afc8] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp, totalSize=36.2 K 2024-11-14T09:16:42,016 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7761a02d17c64f6d874518fb4c179568, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731575795345 2024-11-14T09:16:42,016 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29328a346cf949d2aa4d90a1cd50506c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731575796541 2024-11-14T09:16:42,016 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 624e5540b0214f348a422fc18aa6afc8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731575798746 2024-11-14T09:16:42,019 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/74c355a3333d48e9b6b50f0e75a5a21b as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/74c355a3333d48e9b6b50f0e75a5a21b 2024-11-14T09:16:42,023 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d18b03686124f021810b831ea32e069/A of 1d18b03686124f021810b831ea32e069 into 74c355a3333d48e9b6b50f0e75a5a21b(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:16:42,023 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d18b03686124f021810b831ea32e069: 2024-11-14T09:16:42,023 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., storeName=1d18b03686124f021810b831ea32e069/A, priority=13, startTime=1731575801585; duration=0sec 2024-11-14T09:16:42,023 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:16:42,023 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d18b03686124f021810b831ea32e069:A 2024-11-14T09:16:42,024 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d18b03686124f021810b831ea32e069#C#compaction#500 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:16:42,025 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/4467ee897e7049ff9cb6ae12f363e9da is 50, key is test_row_0/C:col10/1731575798783/Put/seqid=0 2024-11-14T09:16:42,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742419_1595 (size=12915) 2024-11-14T09:16:42,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on 1d18b03686124f021810b831ea32e069 2024-11-14T09:16:42,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. as already flushing 2024-11-14T09:16:42,191 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/6c10e8a0b3d840da90aca0d2b977f664 2024-11-14T09:16:42,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/d8215dfd9f01413191955c5ab715d42c as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/d8215dfd9f01413191955c5ab715d42c 2024-11-14T09:16:42,198 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/d8215dfd9f01413191955c5ab715d42c, entries=150, sequenceid=286, filesize=30.5 K 2024-11-14T09:16:42,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/952d378d76f84d17b56bc68a21326a62 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/952d378d76f84d17b56bc68a21326a62 2024-11-14T09:16:42,202 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/952d378d76f84d17b56bc68a21326a62, entries=150, sequenceid=286, filesize=12.0 K 2024-11-14T09:16:42,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/6c10e8a0b3d840da90aca0d2b977f664 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/6c10e8a0b3d840da90aca0d2b977f664 2024-11-14T09:16:42,206 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/6c10e8a0b3d840da90aca0d2b977f664, entries=150, sequenceid=286, filesize=12.0 K 2024-11-14T09:16:42,207 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 1d18b03686124f021810b831ea32e069 in 455ms, sequenceid=286, compaction requested=false 2024-11-14T09:16:42,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 1d18b03686124f021810b831ea32e069: 2024-11-14T09:16:42,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:42,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-14T09:16:42,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-14T09:16:42,209 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-14T09:16:42,209 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 610 msec 2024-11-14T09:16:42,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on 1d18b03686124f021810b831ea32e069 2024-11-14T09:16:42,210 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1d18b03686124f021810b831ea32e069 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-14T09:16:42,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d18b03686124f021810b831ea32e069, store=A 2024-11-14T09:16:42,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:42,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d18b03686124f021810b831ea32e069, store=B 2024-11-14T09:16:42,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:42,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d18b03686124f021810b831ea32e069, store=C 2024-11-14T09:16:42,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:42,211 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 614 msec 2024-11-14T09:16:42,218 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114595bb88231754f38874e0b0e5d0459da_1d18b03686124f021810b831ea32e069 is 50, key is test_row_0/A:col10/1731575802203/Put/seqid=0 2024-11-14T09:16:42,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742420_1596 (size=14994) 2024-11-14T09:16:42,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-14T09:16:42,225 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:16:42,225 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:16:42,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-14T09:16:42,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-14T09:16:42,227 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:16:42,227 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:16:42,227 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:16:42,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:42,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731575862279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:42,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:42,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:42,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:42,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-14T09:16:42,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-14T09:16:42,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:42,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. as already flushing 2024-11-14T09:16:42,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:42,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:42,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:42,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:42,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:42,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731575862395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:42,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:42,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:42,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:42,432 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/4467ee897e7049ff9cb6ae12f363e9da as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/4467ee897e7049ff9cb6ae12f363e9da 2024-11-14T09:16:42,436 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d18b03686124f021810b831ea32e069/C of 1d18b03686124f021810b831ea32e069 into 4467ee897e7049ff9cb6ae12f363e9da(size=12.6 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:16:42,436 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d18b03686124f021810b831ea32e069: 2024-11-14T09:16:42,436 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., storeName=1d18b03686124f021810b831ea32e069/C, priority=13, startTime=1731575801586; duration=0sec 2024-11-14T09:16:42,436 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:16:42,436 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d18b03686124f021810b831ea32e069:C 2024-11-14T09:16:42,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-14T09:16:42,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:42,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. as already flushing 2024-11-14T09:16:42,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:42,533 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:42,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:42,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:42,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-14T09:16:42,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:42,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731575862605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:42,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:42,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:42,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:42,622 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:42,625 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114595bb88231754f38874e0b0e5d0459da_1d18b03686124f021810b831ea32e069 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114595bb88231754f38874e0b0e5d0459da_1d18b03686124f021810b831ea32e069 2024-11-14T09:16:42,626 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/6295505040cd484aa512f0b86cb278c9, store: [table=TestAcidGuarantees family=A region=1d18b03686124f021810b831ea32e069] 2024-11-14T09:16:42,626 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/6295505040cd484aa512f0b86cb278c9 is 175, key is test_row_0/A:col10/1731575802203/Put/seqid=0 2024-11-14T09:16:42,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742421_1597 (size=39949) 2024-11-14T09:16:42,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-14T09:16:42,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:42,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. as already flushing 2024-11-14T09:16:42,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:42,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:42,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:42,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:42,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-14T09:16:42,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:42,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. as already flushing 2024-11-14T09:16:42,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:42,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:42,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:42,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:42,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-14T09:16:42,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:42,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731575862915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:42,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:42,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:42,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:42,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-14T09:16:42,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:42,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. as already flushing 2024-11-14T09:16:42,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:42,990 ERROR [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:42,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:42,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:16:43,031 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=305, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/6295505040cd484aa512f0b86cb278c9 2024-11-14T09:16:43,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/84b97948925940a3831e1faddac1a304 is 50, key is test_row_0/B:col10/1731575802203/Put/seqid=0 2024-11-14T09:16:43,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742422_1598 (size=12301) 2024-11-14T09:16:43,044 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/84b97948925940a3831e1faddac1a304 2024-11-14T09:16:43,050 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/e5e0791964a04c26bb6cb041048cc4de is 50, key is test_row_0/C:col10/1731575802203/Put/seqid=0 2024-11-14T09:16:43,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742423_1599 (size=12301) 2024-11-14T09:16:43,054 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/e5e0791964a04c26bb6cb041048cc4de 2024-11-14T09:16:43,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/6295505040cd484aa512f0b86cb278c9 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/6295505040cd484aa512f0b86cb278c9 2024-11-14T09:16:43,062 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/6295505040cd484aa512f0b86cb278c9, entries=200, sequenceid=305, filesize=39.0 K 2024-11-14T09:16:43,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/84b97948925940a3831e1faddac1a304 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/84b97948925940a3831e1faddac1a304 2024-11-14T09:16:43,066 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/84b97948925940a3831e1faddac1a304, entries=150, sequenceid=305, filesize=12.0 K 2024-11-14T09:16:43,067 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/e5e0791964a04c26bb6cb041048cc4de as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/e5e0791964a04c26bb6cb041048cc4de 2024-11-14T09:16:43,070 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/e5e0791964a04c26bb6cb041048cc4de, entries=150, sequenceid=305, filesize=12.0 K 2024-11-14T09:16:43,071 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 1d18b03686124f021810b831ea32e069 in 861ms, sequenceid=305, compaction requested=true 2024-11-14T09:16:43,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1d18b03686124f021810b831ea32e069: 2024-11-14T09:16:43,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d18b03686124f021810b831ea32e069:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:16:43,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:16:43,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d18b03686124f021810b831ea32e069:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:16:43,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:16:43,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d18b03686124f021810b831ea32e069:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:16:43,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:16:43,071 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:16:43,071 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:16:43,072 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:16:43,072 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:16:43,072 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): 1d18b03686124f021810b831ea32e069/B is initiating minor compaction (all files) 2024-11-14T09:16:43,072 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): 1d18b03686124f021810b831ea32e069/A is initiating minor compaction (all files) 2024-11-14T09:16:43,072 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d18b03686124f021810b831ea32e069/A in TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:43,072 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d18b03686124f021810b831ea32e069/B in TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:43,072 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/74c355a3333d48e9b6b50f0e75a5a21b, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/d8215dfd9f01413191955c5ab715d42c, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/6295505040cd484aa512f0b86cb278c9] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp, totalSize=100.7 K 2024-11-14T09:16:43,072 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/0fc23249934744688eca69a8a12c9a25, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/952d378d76f84d17b56bc68a21326a62, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/84b97948925940a3831e1faddac1a304] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp, totalSize=36.6 K 2024-11-14T09:16:43,072 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:43,073 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. files: [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/74c355a3333d48e9b6b50f0e75a5a21b, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/d8215dfd9f01413191955c5ab715d42c, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/6295505040cd484aa512f0b86cb278c9] 2024-11-14T09:16:43,073 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fc23249934744688eca69a8a12c9a25, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731575798746 2024-11-14T09:16:43,074 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 952d378d76f84d17b56bc68a21326a62, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731575799967 2024-11-14T09:16:43,074 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74c355a3333d48e9b6b50f0e75a5a21b, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731575798746 2024-11-14T09:16:43,074 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 84b97948925940a3831e1faddac1a304, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731575802198 2024-11-14T09:16:43,074 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8215dfd9f01413191955c5ab715d42c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731575799967 2024-11-14T09:16:43,075 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6295505040cd484aa512f0b86cb278c9, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731575802172 2024-11-14T09:16:43,089 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1d18b03686124f021810b831ea32e069] 2024-11-14T09:16:43,090 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d18b03686124f021810b831ea32e069#B#compaction#504 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:16:43,091 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411146c3082eace144749beaab4e4437c7773_1d18b03686124f021810b831ea32e069 store=[table=TestAcidGuarantees family=A region=1d18b03686124f021810b831ea32e069] 2024-11-14T09:16:43,091 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/def91a845ca44f548cfc552821a1f2ba is 50, key is test_row_0/B:col10/1731575802203/Put/seqid=0 2024-11-14T09:16:43,092 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411146c3082eace144749beaab4e4437c7773_1d18b03686124f021810b831ea32e069, store=[table=TestAcidGuarantees family=A region=1d18b03686124f021810b831ea32e069] 2024-11-14T09:16:43,092 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411146c3082eace144749beaab4e4437c7773_1d18b03686124f021810b831ea32e069 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1d18b03686124f021810b831ea32e069] 2024-11-14T09:16:43,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742424_1600 (size=13017) 2024-11-14T09:16:43,099 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/def91a845ca44f548cfc552821a1f2ba as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/def91a845ca44f548cfc552821a1f2ba 2024-11-14T09:16:43,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742425_1601 (size=4469) 2024-11-14T09:16:43,104 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d18b03686124f021810b831ea32e069/B of 1d18b03686124f021810b831ea32e069 into def91a845ca44f548cfc552821a1f2ba(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:16:43,104 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d18b03686124f021810b831ea32e069: 2024-11-14T09:16:43,104 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., storeName=1d18b03686124f021810b831ea32e069/B, priority=13, startTime=1731575803071; duration=0sec 2024-11-14T09:16:43,104 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:16:43,104 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d18b03686124f021810b831ea32e069:B 2024-11-14T09:16:43,105 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:16:43,105 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:16:43,105 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): 1d18b03686124f021810b831ea32e069/C is initiating minor compaction (all files) 2024-11-14T09:16:43,105 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d18b03686124f021810b831ea32e069/C in TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:43,105 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/4467ee897e7049ff9cb6ae12f363e9da, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/6c10e8a0b3d840da90aca0d2b977f664, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/e5e0791964a04c26bb6cb041048cc4de] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp, totalSize=36.6 K 2024-11-14T09:16:43,106 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 4467ee897e7049ff9cb6ae12f363e9da, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731575798746 2024-11-14T09:16:43,106 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c10e8a0b3d840da90aca0d2b977f664, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731575799967 2024-11-14T09:16:43,106 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting e5e0791964a04c26bb6cb041048cc4de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731575802198 2024-11-14T09:16:43,113 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d18b03686124f021810b831ea32e069#C#compaction#506 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:16:43,113 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/ea3412a382f347ada86d3eef557bbd1d is 50, key is test_row_0/C:col10/1731575802203/Put/seqid=0 2024-11-14T09:16:43,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742426_1602 (size=13017) 2024-11-14T09:16:43,123 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/ea3412a382f347ada86d3eef557bbd1d as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/ea3412a382f347ada86d3eef557bbd1d 2024-11-14T09:16:43,128 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d18b03686124f021810b831ea32e069/C of 1d18b03686124f021810b831ea32e069 into ea3412a382f347ada86d3eef557bbd1d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:16:43,128 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d18b03686124f021810b831ea32e069: 2024-11-14T09:16:43,128 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., storeName=1d18b03686124f021810b831ea32e069/C, priority=13, startTime=1731575803071; duration=0sec 2024-11-14T09:16:43,128 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:16:43,128 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d18b03686124f021810b831ea32e069:C 2024-11-14T09:16:43,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39927 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-14T09:16:43,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:43,143 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 1d18b03686124f021810b831ea32e069 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-14T09:16:43,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d18b03686124f021810b831ea32e069, store=A 2024-11-14T09:16:43,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:43,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d18b03686124f021810b831ea32e069, store=B 2024-11-14T09:16:43,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:43,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d18b03686124f021810b831ea32e069, store=C 2024-11-14T09:16:43,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:43,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114a681086f5f6b4bebb684fcca94d648cf_1d18b03686124f021810b831ea32e069 is 50, key is test_row_0/A:col10/1731575802251/Put/seqid=0 2024-11-14T09:16:43,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742427_1603 (size=12454) 2024-11-14T09:16:43,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-14T09:16:43,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on 1d18b03686124f021810b831ea32e069 2024-11-14T09:16:43,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. as already flushing 2024-11-14T09:16:43,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:43,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731575863482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:43,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:43,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:43,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:43,505 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d18b03686124f021810b831ea32e069#A#compaction#505 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:16:43,506 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/9920a2f00ade44c09f007a1b0f381297 is 175, key is test_row_0/A:col10/1731575802203/Put/seqid=0 2024-11-14T09:16:43,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742428_1604 (size=31971) 2024-11-14T09:16:43,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:43,560 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114a681086f5f6b4bebb684fcca94d648cf_1d18b03686124f021810b831ea32e069 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114a681086f5f6b4bebb684fcca94d648cf_1d18b03686124f021810b831ea32e069 2024-11-14T09:16:43,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/3a334c14f66144c3bd4ec3693b168a18, store: [table=TestAcidGuarantees family=A region=1d18b03686124f021810b831ea32e069] 2024-11-14T09:16:43,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/3a334c14f66144c3bd4ec3693b168a18 is 175, key is test_row_0/A:col10/1731575802251/Put/seqid=0 2024-11-14T09:16:43,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742429_1605 (size=31255) 2024-11-14T09:16:43,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:43,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731575863595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:43,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:43,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:43,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:43,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:43,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731575863805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:43,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:43,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:43,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:43,913 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/9920a2f00ade44c09f007a1b0f381297 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/9920a2f00ade44c09f007a1b0f381297 2024-11-14T09:16:43,917 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d18b03686124f021810b831ea32e069/A of 1d18b03686124f021810b831ea32e069 into 9920a2f00ade44c09f007a1b0f381297(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:16:43,917 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d18b03686124f021810b831ea32e069: 2024-11-14T09:16:43,918 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., storeName=1d18b03686124f021810b831ea32e069/A, priority=13, startTime=1731575803071; duration=0sec 2024-11-14T09:16:43,918 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:16:43,918 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d18b03686124f021810b831ea32e069:A 2024-11-14T09:16:43,966 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=323, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/3a334c14f66144c3bd4ec3693b168a18 2024-11-14T09:16:43,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/6320ad2944534ada9cba8433217e8c6e is 50, key is test_row_0/B:col10/1731575802251/Put/seqid=0 2024-11-14T09:16:43,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742430_1606 (size=12301) 2024-11-14T09:16:44,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:16:44,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731575864114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:44,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 , the old value is region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:44,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:16:44,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., hostname=83f56b55f2af,39927,1731575622393, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:16:44,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-14T09:16:44,376 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/6320ad2944534ada9cba8433217e8c6e 2024-11-14T09:16:44,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/e6e35d5bdd02408082798a796bdf7b42 is 50, key is test_row_0/C:col10/1731575802251/Put/seqid=0 2024-11-14T09:16:44,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742431_1607 (size=12301) 2024-11-14T09:16:44,398 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/e6e35d5bdd02408082798a796bdf7b42 2024-11-14T09:16:44,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/3a334c14f66144c3bd4ec3693b168a18 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/3a334c14f66144c3bd4ec3693b168a18 2024-11-14T09:16:44,405 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/3a334c14f66144c3bd4ec3693b168a18, entries=150, sequenceid=323, filesize=30.5 K 2024-11-14T09:16:44,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/6320ad2944534ada9cba8433217e8c6e as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/6320ad2944534ada9cba8433217e8c6e 2024-11-14T09:16:44,409 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/6320ad2944534ada9cba8433217e8c6e, entries=150, sequenceid=323, filesize=12.0 K 2024-11-14T09:16:44,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/e6e35d5bdd02408082798a796bdf7b42 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/e6e35d5bdd02408082798a796bdf7b42 2024-11-14T09:16:44,412 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/e6e35d5bdd02408082798a796bdf7b42, entries=150, sequenceid=323, filesize=12.0 K 2024-11-14T09:16:44,413 INFO [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 1d18b03686124f021810b831ea32e069 in 1270ms, sequenceid=323, compaction requested=false 2024-11-14T09:16:44,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 1d18b03686124f021810b831ea32e069: 2024-11-14T09:16:44,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:44,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-14T09:16:44,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-14T09:16:44,416 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-14T09:16:44,416 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1870 sec 2024-11-14T09:16:44,418 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 2.1920 sec 2024-11-14T09:16:44,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on 1d18b03686124f021810b831ea32e069 2024-11-14T09:16:44,628 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1d18b03686124f021810b831ea32e069 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-14T09:16:44,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d18b03686124f021810b831ea32e069, store=A 2024-11-14T09:16:44,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:44,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d18b03686124f021810b831ea32e069, store=B 2024-11-14T09:16:44,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:44,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d18b03686124f021810b831ea32e069, store=C 2024-11-14T09:16:44,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:44,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114857a64ad006b4b23984e336f0269b4f0_1d18b03686124f021810b831ea32e069 is 50, key is test_row_0/A:col10/1731575803472/Put/seqid=0 2024-11-14T09:16:44,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742432_1608 (size=14994) 2024-11-14T09:16:44,639 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:44,642 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114857a64ad006b4b23984e336f0269b4f0_1d18b03686124f021810b831ea32e069 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114857a64ad006b4b23984e336f0269b4f0_1d18b03686124f021810b831ea32e069 2024-11-14T09:16:44,643 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/ca2ef8bb98664ca8a0d85d862bbfb3f7, store: [table=TestAcidGuarantees family=A region=1d18b03686124f021810b831ea32e069] 2024-11-14T09:16:44,643 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/ca2ef8bb98664ca8a0d85d862bbfb3f7 is 175, key is test_row_0/A:col10/1731575803472/Put/seqid=0 2024-11-14T09:16:44,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742433_1609 (size=39949) 2024-11-14T09:16:44,665 INFO [Thread-2329 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2329. 2024-11-14T09:16:44,665 DEBUG [Thread-2329 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:16:44,665 DEBUG [Thread-2329 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:16:44,665 DEBUG [Thread-2329 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:16:44,666 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:16:44,666 INFO [Thread-2320 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2320. 2024-11-14T09:16:44,666 DEBUG [Thread-2320 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:16:44,666 INFO [Thread-2325 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2325. 2024-11-14T09:16:44,666 DEBUG [Thread-2320 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:16:44,666 DEBUG [Thread-2325 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:16:44,666 DEBUG [Thread-2325 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:16:44,666 DEBUG [Thread-2320 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:16:44,666 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:16:44,666 DEBUG [Thread-2325 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:16:44,666 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:16:44,668 INFO [Thread-2327 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2327. 2024-11-14T09:16:44,668 DEBUG [Thread-2327 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:16:44,668 DEBUG [Thread-2327 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:16:44,668 DEBUG [Thread-2327 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:16:44,668 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:16:44,669 INFO [Thread-2333 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2333. 2024-11-14T09:16:44,669 DEBUG [Thread-2333 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:16:44,669 DEBUG [Thread-2333 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:16:44,669 DEBUG [Thread-2333 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:16:44,669 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:16:44,673 INFO [Thread-2331 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2331. 2024-11-14T09:16:44,673 DEBUG [Thread-2331 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:16:44,673 DEBUG [Thread-2331 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:16:44,673 DEBUG [Thread-2331 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:16:44,673 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:16:45,047 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=342, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/ca2ef8bb98664ca8a0d85d862bbfb3f7 2024-11-14T09:16:45,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/3687b0b6fd664999834adc8a9657938d is 50, key is test_row_0/B:col10/1731575803472/Put/seqid=0 2024-11-14T09:16:45,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742434_1610 (size=12301) 2024-11-14T09:16:45,459 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/3687b0b6fd664999834adc8a9657938d 2024-11-14T09:16:45,464 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/e0614f3e9d1349be8d1a7b14483c49b0 is 50, key is test_row_0/C:col10/1731575803472/Put/seqid=0 2024-11-14T09:16:45,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742435_1611 (size=12301) 2024-11-14T09:16:45,873 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/e0614f3e9d1349be8d1a7b14483c49b0 2024-11-14T09:16:45,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/ca2ef8bb98664ca8a0d85d862bbfb3f7 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/ca2ef8bb98664ca8a0d85d862bbfb3f7 2024-11-14T09:16:45,879 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/ca2ef8bb98664ca8a0d85d862bbfb3f7, entries=200, sequenceid=342, filesize=39.0 K 2024-11-14T09:16:45,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/3687b0b6fd664999834adc8a9657938d as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/3687b0b6fd664999834adc8a9657938d 2024-11-14T09:16:45,898 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/3687b0b6fd664999834adc8a9657938d, entries=150, sequenceid=342, filesize=12.0 K 2024-11-14T09:16:45,899 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/e0614f3e9d1349be8d1a7b14483c49b0 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/e0614f3e9d1349be8d1a7b14483c49b0 2024-11-14T09:16:45,902 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/e0614f3e9d1349be8d1a7b14483c49b0, entries=150, sequenceid=342, filesize=12.0 K 2024-11-14T09:16:45,903 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=67.09 KB/68700 for 1d18b03686124f021810b831ea32e069 in 1275ms, sequenceid=342, compaction requested=true 2024-11-14T09:16:45,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1d18b03686124f021810b831ea32e069: 2024-11-14T09:16:45,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d18b03686124f021810b831ea32e069:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:16:45,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:16:45,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d18b03686124f021810b831ea32e069:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:16:45,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:16:45,903 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:16:45,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d18b03686124f021810b831ea32e069:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:16:45,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T09:16:45,903 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:16:45,904 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103175 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:16:45,904 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:16:45,904 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): 1d18b03686124f021810b831ea32e069/A is initiating minor compaction (all files) 2024-11-14T09:16:45,904 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1541): 1d18b03686124f021810b831ea32e069/B is initiating minor compaction (all files) 2024-11-14T09:16:45,904 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d18b03686124f021810b831ea32e069/A in TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:45,904 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d18b03686124f021810b831ea32e069/B in TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:45,904 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/def91a845ca44f548cfc552821a1f2ba, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/6320ad2944534ada9cba8433217e8c6e, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/3687b0b6fd664999834adc8a9657938d] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp, totalSize=36.7 K 2024-11-14T09:16:45,904 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/9920a2f00ade44c09f007a1b0f381297, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/3a334c14f66144c3bd4ec3693b168a18, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/ca2ef8bb98664ca8a0d85d862bbfb3f7] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp, totalSize=100.8 K 2024-11-14T09:16:45,904 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:45,904 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. files: [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/9920a2f00ade44c09f007a1b0f381297, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/3a334c14f66144c3bd4ec3693b168a18, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/ca2ef8bb98664ca8a0d85d862bbfb3f7] 2024-11-14T09:16:45,905 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting def91a845ca44f548cfc552821a1f2ba, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731575802198 2024-11-14T09:16:45,905 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9920a2f00ade44c09f007a1b0f381297, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731575802198 2024-11-14T09:16:45,905 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 6320ad2944534ada9cba8433217e8c6e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731575802251 2024-11-14T09:16:45,905 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a334c14f66144c3bd4ec3693b168a18, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731575802251 2024-11-14T09:16:45,905 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] compactions.Compactor(225): Compacting 3687b0b6fd664999834adc8a9657938d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731575803472 2024-11-14T09:16:45,905 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca2ef8bb98664ca8a0d85d862bbfb3f7, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731575803472 2024-11-14T09:16:45,915 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1d18b03686124f021810b831ea32e069] 2024-11-14T09:16:45,916 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d18b03686124f021810b831ea32e069#B#compaction#513 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:16:45,916 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/9b51808d2eec4b36af6d916f5f0763a9 is 50, key is test_row_0/B:col10/1731575803472/Put/seqid=0 2024-11-14T09:16:45,917 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411142aab1e3a7ae34463b1221f2e8e557ca1_1d18b03686124f021810b831ea32e069 store=[table=TestAcidGuarantees family=A region=1d18b03686124f021810b831ea32e069] 2024-11-14T09:16:45,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742436_1612 (size=13119) 2024-11-14T09:16:45,942 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411142aab1e3a7ae34463b1221f2e8e557ca1_1d18b03686124f021810b831ea32e069, store=[table=TestAcidGuarantees family=A region=1d18b03686124f021810b831ea32e069] 2024-11-14T09:16:45,942 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411142aab1e3a7ae34463b1221f2e8e557ca1_1d18b03686124f021810b831ea32e069 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1d18b03686124f021810b831ea32e069] 2024-11-14T09:16:45,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742437_1613 (size=4469) 2024-11-14T09:16:45,947 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d18b03686124f021810b831ea32e069#A#compaction#514 average throughput is 0.76 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:16:45,947 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/3a18ef6444b84f6c8ce028943927e5d7 is 175, key is test_row_0/A:col10/1731575803472/Put/seqid=0 2024-11-14T09:16:45,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742438_1614 (size=32073) 2024-11-14T09:16:45,956 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/3a18ef6444b84f6c8ce028943927e5d7 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/3a18ef6444b84f6c8ce028943927e5d7 2024-11-14T09:16:45,960 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d18b03686124f021810b831ea32e069/A of 1d18b03686124f021810b831ea32e069 into 3a18ef6444b84f6c8ce028943927e5d7(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:16:45,961 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d18b03686124f021810b831ea32e069: 2024-11-14T09:16:45,961 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., storeName=1d18b03686124f021810b831ea32e069/A, priority=13, startTime=1731575805903; duration=0sec 2024-11-14T09:16:45,961 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:16:45,961 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d18b03686124f021810b831ea32e069:A 2024-11-14T09:16:45,961 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:16:45,962 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:16:45,962 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1541): 1d18b03686124f021810b831ea32e069/C is initiating minor compaction (all files) 2024-11-14T09:16:45,962 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d18b03686124f021810b831ea32e069/C in TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:45,962 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/ea3412a382f347ada86d3eef557bbd1d, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/e6e35d5bdd02408082798a796bdf7b42, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/e0614f3e9d1349be8d1a7b14483c49b0] into tmpdir=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp, totalSize=36.7 K 2024-11-14T09:16:45,962 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea3412a382f347ada86d3eef557bbd1d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731575802198 2024-11-14T09:16:45,962 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6e35d5bdd02408082798a796bdf7b42, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731575802251 2024-11-14T09:16:45,963 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0614f3e9d1349be8d1a7b14483c49b0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731575803472 2024-11-14T09:16:45,996 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d18b03686124f021810b831ea32e069#C#compaction#515 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:16:45,996 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/2399542321c849f5936c7fd11ab38b83 is 50, key is test_row_0/C:col10/1731575803472/Put/seqid=0 2024-11-14T09:16:46,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742439_1615 (size=13119) 2024-11-14T09:16:46,033 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/2399542321c849f5936c7fd11ab38b83 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/2399542321c849f5936c7fd11ab38b83 2024-11-14T09:16:46,047 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d18b03686124f021810b831ea32e069/C of 1d18b03686124f021810b831ea32e069 into 2399542321c849f5936c7fd11ab38b83(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:16:46,047 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d18b03686124f021810b831ea32e069: 2024-11-14T09:16:46,047 INFO [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., storeName=1d18b03686124f021810b831ea32e069/C, priority=13, startTime=1731575805903; duration=0sec 2024-11-14T09:16:46,047 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:16:46,047 DEBUG [RS:0;83f56b55f2af:39927-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d18b03686124f021810b831ea32e069:C 2024-11-14T09:16:46,327 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/9b51808d2eec4b36af6d916f5f0763a9 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/9b51808d2eec4b36af6d916f5f0763a9 2024-11-14T09:16:46,331 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d18b03686124f021810b831ea32e069/B of 1d18b03686124f021810b831ea32e069 into 9b51808d2eec4b36af6d916f5f0763a9(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:16:46,331 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d18b03686124f021810b831ea32e069: 2024-11-14T09:16:46,331 INFO [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., storeName=1d18b03686124f021810b831ea32e069/B, priority=13, startTime=1731575805903; duration=0sec 2024-11-14T09:16:46,331 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:16:46,331 DEBUG [RS:0;83f56b55f2af:39927-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d18b03686124f021810b831ea32e069:B 2024-11-14T09:16:46,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-14T09:16:46,396 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:16:47,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39927 {}] regionserver.HRegion(8855): Flush requested on 1d18b03686124f021810b831ea32e069 2024-11-14T09:16:47,046 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1d18b03686124f021810b831ea32e069 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-14T09:16:47,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d18b03686124f021810b831ea32e069, store=A 2024-11-14T09:16:47,046 INFO [Thread-2316 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2316. 2024-11-14T09:16:47,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:47,046 DEBUG [Thread-2316 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:16:47,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d18b03686124f021810b831ea32e069, store=B 2024-11-14T09:16:47,046 DEBUG [Thread-2316 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:16:47,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:47,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d18b03686124f021810b831ea32e069, store=C 2024-11-14T09:16:47,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:47,047 DEBUG [Thread-2316 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:16:47,047 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:16:47,051 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114835d2a83978145a18f21794f517f8d86_1d18b03686124f021810b831ea32e069 is 50, key is test_row_0/A:col10/1731575807045/Put/seqid=0 2024-11-14T09:16:47,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742440_1616 (size=12454) 2024-11-14T09:16:47,455 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:47,458 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114835d2a83978145a18f21794f517f8d86_1d18b03686124f021810b831ea32e069 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114835d2a83978145a18f21794f517f8d86_1d18b03686124f021810b831ea32e069 2024-11-14T09:16:47,458 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/f71899fd38eb43b69fe008bce9a3b7af, store: [table=TestAcidGuarantees family=A region=1d18b03686124f021810b831ea32e069] 2024-11-14T09:16:47,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/f71899fd38eb43b69fe008bce9a3b7af is 175, key is test_row_0/A:col10/1731575807045/Put/seqid=0 2024-11-14T09:16:47,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742441_1617 (size=31255) 2024-11-14T09:16:47,696 INFO [Thread-2314 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2314. 2024-11-14T09:16:47,696 INFO [Thread-2322 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2322. 2024-11-14T09:16:47,697 DEBUG [Thread-2322 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:16:47,697 DEBUG [Thread-2314 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:16:47,697 DEBUG [Thread-2322 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:16:47,697 DEBUG [Thread-2314 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:16:47,697 DEBUG [Thread-2314 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:16:47,697 DEBUG [Thread-2322 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:16:47,697 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:16:47,697 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:16:47,706 INFO [Thread-2318 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2318. 2024-11-14T09:16:47,706 DEBUG [Thread-2318 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:16:47,706 DEBUG [Thread-2318 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:16:47,706 DEBUG [Thread-2318 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:16:47,706 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:16:47,707 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T09:16:47,707 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 27 2024-11-14T09:16:47,707 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 23 2024-11-14T09:16:47,707 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 29 2024-11-14T09:16:47,707 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 175 2024-11-14T09:16:47,707 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 26 2024-11-14T09:16:47,707 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T09:16:47,707 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2994 2024-11-14T09:16:47,707 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2975 2024-11-14T09:16:47,707 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2963 2024-11-14T09:16:47,707 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2993 2024-11-14T09:16:47,707 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2971 2024-11-14T09:16:47,707 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T09:16:47,707 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T09:16:47,707 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T09:16:47,707 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:16:47,707 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:16:47,707 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:16:47,708 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-14T09:16:47,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T09:16:47,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-14T09:16:47,710 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731575807710"}]},"ts":"1731575807710"} 2024-11-14T09:16:47,711 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T09:16:47,711 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T09:16:47,712 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T09:16:47,713 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d18b03686124f021810b831ea32e069, UNASSIGN}] 2024-11-14T09:16:47,713 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d18b03686124f021810b831ea32e069, UNASSIGN 2024-11-14T09:16:47,714 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=162 updating hbase:meta row=1d18b03686124f021810b831ea32e069, regionState=CLOSING, regionLocation=83f56b55f2af,39927,1731575622393 2024-11-14T09:16:47,715 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d18b03686124f021810b831ea32e069, UNASSIGN because future has completed 2024-11-14T09:16:47,715 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T09:16:47,715 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393}] 2024-11-14T09:16:47,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-14T09:16:47,863 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=359, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/f71899fd38eb43b69fe008bce9a3b7af 2024-11-14T09:16:47,867 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] handler.UnassignRegionHandler(122): Close 1d18b03686124f021810b831ea32e069 2024-11-14T09:16:47,867 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T09:16:47,867 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1722): Closing 1d18b03686124f021810b831ea32e069, disabling compactions & flushes 2024-11-14T09:16:47,867 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:47,868 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/eef6d62dc3724fa3928434d03851c5ec is 50, key is test_row_0/B:col10/1731575807045/Put/seqid=0 2024-11-14T09:16:47,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742442_1618 (size=12301) 2024-11-14T09:16:48,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-14T09:16:48,272 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/eef6d62dc3724fa3928434d03851c5ec 2024-11-14T09:16:48,277 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/80fea942f1a0451cb668a0c980e5e272 is 50, key is test_row_0/C:col10/1731575807045/Put/seqid=0 2024-11-14T09:16:48,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742443_1619 (size=12301) 2024-11-14T09:16:48,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-14T09:16:48,680 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/80fea942f1a0451cb668a0c980e5e272 2024-11-14T09:16:48,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/f71899fd38eb43b69fe008bce9a3b7af as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/f71899fd38eb43b69fe008bce9a3b7af 2024-11-14T09:16:48,686 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/f71899fd38eb43b69fe008bce9a3b7af, entries=150, sequenceid=359, filesize=30.5 K 2024-11-14T09:16:48,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/eef6d62dc3724fa3928434d03851c5ec as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/eef6d62dc3724fa3928434d03851c5ec 2024-11-14T09:16:48,689 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/eef6d62dc3724fa3928434d03851c5ec, entries=150, sequenceid=359, filesize=12.0 K 2024-11-14T09:16:48,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/80fea942f1a0451cb668a0c980e5e272 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/80fea942f1a0451cb668a0c980e5e272 2024-11-14T09:16:48,692 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/80fea942f1a0451cb668a0c980e5e272, entries=150, sequenceid=359, filesize=12.0 K 2024-11-14T09:16:48,692 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=20.13 KB/20610 for 1d18b03686124f021810b831ea32e069 in 1646ms, sequenceid=359, compaction requested=false 2024-11-14T09:16:48,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1d18b03686124f021810b831ea32e069: 2024-11-14T09:16:48,692 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:48,692 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:48,692 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. after waiting 0 ms 2024-11-14T09:16:48,692 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:48,693 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(2902): Flushing 1d18b03686124f021810b831ea32e069 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-14T09:16:48,693 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d18b03686124f021810b831ea32e069, store=A 2024-11-14T09:16:48,693 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:48,693 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d18b03686124f021810b831ea32e069, store=B 2024-11-14T09:16:48,693 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:48,693 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d18b03686124f021810b831ea32e069, store=C 2024-11-14T09:16:48,693 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:16:48,697 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114dbade5bc05ae4d39b2cec94a7390d244_1d18b03686124f021810b831ea32e069 is 50, key is test_row_0/A:col10/1731575807705/Put/seqid=0 2024-11-14T09:16:48,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742444_1620 (size=9914) 2024-11-14T09:16:48,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-14T09:16:49,100 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:16:49,103 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114dbade5bc05ae4d39b2cec94a7390d244_1d18b03686124f021810b831ea32e069 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114dbade5bc05ae4d39b2cec94a7390d244_1d18b03686124f021810b831ea32e069 2024-11-14T09:16:49,104 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/d54662451e0f4fb190358d048582f9ca, store: [table=TestAcidGuarantees family=A region=1d18b03686124f021810b831ea32e069] 2024-11-14T09:16:49,105 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/d54662451e0f4fb190358d048582f9ca is 175, key is test_row_0/A:col10/1731575807705/Put/seqid=0 2024-11-14T09:16:49,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742445_1621 (size=22561) 2024-11-14T09:16:49,508 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=365, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/d54662451e0f4fb190358d048582f9ca 2024-11-14T09:16:49,513 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/ecf9431e33b54e63841b623e96ae7e3e is 50, key is test_row_0/B:col10/1731575807705/Put/seqid=0 2024-11-14T09:16:49,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742446_1622 (size=9857) 2024-11-14T09:16:49,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-14T09:16:49,917 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/ecf9431e33b54e63841b623e96ae7e3e 2024-11-14T09:16:49,922 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/924d4d059921457488e4b754fc266eab is 50, key is test_row_0/C:col10/1731575807705/Put/seqid=0 2024-11-14T09:16:49,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742447_1623 (size=9857) 2024-11-14T09:16:50,326 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/924d4d059921457488e4b754fc266eab 2024-11-14T09:16:50,329 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/A/d54662451e0f4fb190358d048582f9ca as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/d54662451e0f4fb190358d048582f9ca 2024-11-14T09:16:50,331 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/d54662451e0f4fb190358d048582f9ca, entries=100, sequenceid=365, filesize=22.0 K 2024-11-14T09:16:50,332 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/B/ecf9431e33b54e63841b623e96ae7e3e as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/ecf9431e33b54e63841b623e96ae7e3e 2024-11-14T09:16:50,334 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/ecf9431e33b54e63841b623e96ae7e3e, entries=100, sequenceid=365, filesize=9.6 K 2024-11-14T09:16:50,335 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/.tmp/C/924d4d059921457488e4b754fc266eab as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/924d4d059921457488e4b754fc266eab 2024-11-14T09:16:50,337 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/924d4d059921457488e4b754fc266eab, entries=100, sequenceid=365, filesize=9.6 K 2024-11-14T09:16:50,338 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 1d18b03686124f021810b831ea32e069 in 1646ms, sequenceid=365, compaction requested=true 2024-11-14T09:16:50,338 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/cdb74091cb0c43338a5bbaa1a49955d0, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/924d28422b2443b3a0ecc9735ba1c183, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/22412f2db5a74503972db6f733710009, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/ce0647ea93ed4f409af189067f47d9d8, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/221464a389b240e0981ba6864d45c130, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/778813b64a414e6198ed7beba49f09e7, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/163956fbdef6422486d02fda8b5ecb91, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/19893271ff0640daa58ffe84312b425d, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/926838a9c6bb447fb1e088fbe7d0c918, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/c12bb0e24fa24ca988d909d9d5b457f2, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/ed08a05d99ca40769f9b6bb4e4d0271f, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/5b0e04409c234d1ca07cb2a08365cf8b, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/dd4aada6a1944c28b77b4d71ee5c5743, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/96336ee5892a49e3a482d68db457fc5e, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/2f33cc2520c047979b34a7c9a752c1eb, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/3b74aebb23684c8abbc886bf240a8e10, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/09a1ccd087574c268c8bf8c6c326a798, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/c0eb6508740c47ba84320e5e096577e5, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/74c355a3333d48e9b6b50f0e75a5a21b, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/d8215dfd9f01413191955c5ab715d42c, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/6295505040cd484aa512f0b86cb278c9, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/9920a2f00ade44c09f007a1b0f381297, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/3a334c14f66144c3bd4ec3693b168a18, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/ca2ef8bb98664ca8a0d85d862bbfb3f7] to archive 2024-11-14T09:16:50,339 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T09:16:50,340 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/cdb74091cb0c43338a5bbaa1a49955d0 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/cdb74091cb0c43338a5bbaa1a49955d0 2024-11-14T09:16:50,341 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/924d28422b2443b3a0ecc9735ba1c183 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/924d28422b2443b3a0ecc9735ba1c183 2024-11-14T09:16:50,342 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/22412f2db5a74503972db6f733710009 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/22412f2db5a74503972db6f733710009 2024-11-14T09:16:50,342 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/ce0647ea93ed4f409af189067f47d9d8 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/ce0647ea93ed4f409af189067f47d9d8 2024-11-14T09:16:50,343 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/221464a389b240e0981ba6864d45c130 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/221464a389b240e0981ba6864d45c130 2024-11-14T09:16:50,344 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/778813b64a414e6198ed7beba49f09e7 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/778813b64a414e6198ed7beba49f09e7 2024-11-14T09:16:50,345 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/163956fbdef6422486d02fda8b5ecb91 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/163956fbdef6422486d02fda8b5ecb91 2024-11-14T09:16:50,346 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/19893271ff0640daa58ffe84312b425d to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/19893271ff0640daa58ffe84312b425d 2024-11-14T09:16:50,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/926838a9c6bb447fb1e088fbe7d0c918 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/926838a9c6bb447fb1e088fbe7d0c918 2024-11-14T09:16:50,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/c12bb0e24fa24ca988d909d9d5b457f2 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/c12bb0e24fa24ca988d909d9d5b457f2 2024-11-14T09:16:50,348 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/ed08a05d99ca40769f9b6bb4e4d0271f to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/ed08a05d99ca40769f9b6bb4e4d0271f 2024-11-14T09:16:50,349 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/5b0e04409c234d1ca07cb2a08365cf8b to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/5b0e04409c234d1ca07cb2a08365cf8b 2024-11-14T09:16:50,350 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/dd4aada6a1944c28b77b4d71ee5c5743 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/dd4aada6a1944c28b77b4d71ee5c5743 2024-11-14T09:16:50,350 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/96336ee5892a49e3a482d68db457fc5e to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/96336ee5892a49e3a482d68db457fc5e 2024-11-14T09:16:50,351 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/2f33cc2520c047979b34a7c9a752c1eb to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/2f33cc2520c047979b34a7c9a752c1eb 2024-11-14T09:16:50,352 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/3b74aebb23684c8abbc886bf240a8e10 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/3b74aebb23684c8abbc886bf240a8e10 2024-11-14T09:16:50,353 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/09a1ccd087574c268c8bf8c6c326a798 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/09a1ccd087574c268c8bf8c6c326a798 2024-11-14T09:16:50,353 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/c0eb6508740c47ba84320e5e096577e5 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/c0eb6508740c47ba84320e5e096577e5 2024-11-14T09:16:50,354 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/74c355a3333d48e9b6b50f0e75a5a21b to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/74c355a3333d48e9b6b50f0e75a5a21b 2024-11-14T09:16:50,355 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/d8215dfd9f01413191955c5ab715d42c to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/d8215dfd9f01413191955c5ab715d42c 2024-11-14T09:16:50,356 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/6295505040cd484aa512f0b86cb278c9 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/6295505040cd484aa512f0b86cb278c9 2024-11-14T09:16:50,356 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/9920a2f00ade44c09f007a1b0f381297 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/9920a2f00ade44c09f007a1b0f381297 2024-11-14T09:16:50,357 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/3a334c14f66144c3bd4ec3693b168a18 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/3a334c14f66144c3bd4ec3693b168a18 2024-11-14T09:16:50,358 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/ca2ef8bb98664ca8a0d85d862bbfb3f7 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/ca2ef8bb98664ca8a0d85d862bbfb3f7 2024-11-14T09:16:50,359 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/abb0b3279ea14c4797b2b2a3c1951ff7, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/b1277aac80824eceab1c93889bc13d1d, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/55b3d9bd72a240859ad7f4748b5f94fd, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/33b69277391344499c80bea12591dd72, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/e852e42d6c8b4dc5823ff8b289427ee5, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/82f827e7937d44ebbc0813a6037841e6, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/cc42c32a131b402b93887ef1e5998ad2, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/ff7c1be6866541c0ad90fa5b0e261ae5, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/b00111888185487c94bbd140cf7406a7, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/70c05adab43340ce88cdc1aaca83f793, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/2709a094ac9b461c9aca8d4017c244ce, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/ba3198e507eb4693a2bbde1bf3ff4a93, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/af62a1c0ac4a491bb8a2ed00a1ff14d0, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/c24c5a3f8fcc49f98a109563330f6a5e, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/15271176d80d4702ae1077a7d44726ad, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/69e91c71adf64f3982864a845b629369, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/361ac23c0e094cb3a7325693742c02f5, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/0fc23249934744688eca69a8a12c9a25, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/b889837572fb40f7b8d5f72c1f7c39a7, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/952d378d76f84d17b56bc68a21326a62, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/def91a845ca44f548cfc552821a1f2ba, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/84b97948925940a3831e1faddac1a304, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/6320ad2944534ada9cba8433217e8c6e, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/3687b0b6fd664999834adc8a9657938d] to archive 2024-11-14T09:16:50,360 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T09:16:50,361 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/abb0b3279ea14c4797b2b2a3c1951ff7 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/abb0b3279ea14c4797b2b2a3c1951ff7 2024-11-14T09:16:50,362 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/b1277aac80824eceab1c93889bc13d1d to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/b1277aac80824eceab1c93889bc13d1d 2024-11-14T09:16:50,362 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/55b3d9bd72a240859ad7f4748b5f94fd to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/55b3d9bd72a240859ad7f4748b5f94fd 2024-11-14T09:16:50,363 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/33b69277391344499c80bea12591dd72 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/33b69277391344499c80bea12591dd72 2024-11-14T09:16:50,364 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/e852e42d6c8b4dc5823ff8b289427ee5 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/e852e42d6c8b4dc5823ff8b289427ee5 2024-11-14T09:16:50,365 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/82f827e7937d44ebbc0813a6037841e6 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/82f827e7937d44ebbc0813a6037841e6 2024-11-14T09:16:50,366 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/cc42c32a131b402b93887ef1e5998ad2 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/cc42c32a131b402b93887ef1e5998ad2 2024-11-14T09:16:50,367 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/ff7c1be6866541c0ad90fa5b0e261ae5 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/ff7c1be6866541c0ad90fa5b0e261ae5 2024-11-14T09:16:50,368 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/b00111888185487c94bbd140cf7406a7 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/b00111888185487c94bbd140cf7406a7 2024-11-14T09:16:50,368 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/70c05adab43340ce88cdc1aaca83f793 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/70c05adab43340ce88cdc1aaca83f793 2024-11-14T09:16:50,369 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/2709a094ac9b461c9aca8d4017c244ce to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/2709a094ac9b461c9aca8d4017c244ce 2024-11-14T09:16:50,370 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/ba3198e507eb4693a2bbde1bf3ff4a93 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/ba3198e507eb4693a2bbde1bf3ff4a93 2024-11-14T09:16:50,371 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/af62a1c0ac4a491bb8a2ed00a1ff14d0 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/af62a1c0ac4a491bb8a2ed00a1ff14d0 2024-11-14T09:16:50,371 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/c24c5a3f8fcc49f98a109563330f6a5e to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/c24c5a3f8fcc49f98a109563330f6a5e 2024-11-14T09:16:50,372 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/15271176d80d4702ae1077a7d44726ad to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/15271176d80d4702ae1077a7d44726ad 2024-11-14T09:16:50,373 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/69e91c71adf64f3982864a845b629369 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/69e91c71adf64f3982864a845b629369 2024-11-14T09:16:50,374 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/361ac23c0e094cb3a7325693742c02f5 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/361ac23c0e094cb3a7325693742c02f5 2024-11-14T09:16:50,374 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/0fc23249934744688eca69a8a12c9a25 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/0fc23249934744688eca69a8a12c9a25 2024-11-14T09:16:50,375 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/b889837572fb40f7b8d5f72c1f7c39a7 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/b889837572fb40f7b8d5f72c1f7c39a7 2024-11-14T09:16:50,376 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/952d378d76f84d17b56bc68a21326a62 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/952d378d76f84d17b56bc68a21326a62 2024-11-14T09:16:50,376 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/def91a845ca44f548cfc552821a1f2ba to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/def91a845ca44f548cfc552821a1f2ba 2024-11-14T09:16:50,377 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/84b97948925940a3831e1faddac1a304 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/84b97948925940a3831e1faddac1a304 2024-11-14T09:16:50,378 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/6320ad2944534ada9cba8433217e8c6e to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/6320ad2944534ada9cba8433217e8c6e 2024-11-14T09:16:50,378 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/3687b0b6fd664999834adc8a9657938d to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/3687b0b6fd664999834adc8a9657938d 2024-11-14T09:16:50,380 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/4f3bf9b5135e4f20b04059ed9aa8c7fa, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/8d41bb25d20245718e2e0075b2c38301, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/ad2589148e8d4c939b5c46acf98b6a32, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/ccfe26afa43442ab99e961a052ca969d, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/ee8b9b11c5814d3bbc1a3d54b4ac8a2b, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/98b4639f19fd48a195cef8d62878dcf7, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/3313320d349d453198100d89ec9ce8c1, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/5c55161fd10e4bd5a82bacc8faba0bb4, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/54b3756177fd4d8281dc31d657ef607d, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/1cb2c63c4888404d8e0b5ff435f300c0, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/d54e098e94ef4e45a1f1eb3e9f5dbe3c, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/0e2fd4c37d104ae693123f54b2437055, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/f7e065fac6364702a452ef75d6dc0121, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/ff79faf6140a421cb4796ac25a9faacc, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/7761a02d17c64f6d874518fb4c179568, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/420b9f987921497e979e00b0e68eb8e4, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/29328a346cf949d2aa4d90a1cd50506c, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/4467ee897e7049ff9cb6ae12f363e9da, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/624e5540b0214f348a422fc18aa6afc8, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/6c10e8a0b3d840da90aca0d2b977f664, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/ea3412a382f347ada86d3eef557bbd1d, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/e5e0791964a04c26bb6cb041048cc4de, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/e6e35d5bdd02408082798a796bdf7b42, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/e0614f3e9d1349be8d1a7b14483c49b0] to archive 2024-11-14T09:16:50,380 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T09:16:50,381 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/4f3bf9b5135e4f20b04059ed9aa8c7fa to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/4f3bf9b5135e4f20b04059ed9aa8c7fa 2024-11-14T09:16:50,382 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/8d41bb25d20245718e2e0075b2c38301 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/8d41bb25d20245718e2e0075b2c38301 2024-11-14T09:16:50,382 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/ad2589148e8d4c939b5c46acf98b6a32 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/ad2589148e8d4c939b5c46acf98b6a32 2024-11-14T09:16:50,383 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/ccfe26afa43442ab99e961a052ca969d to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/ccfe26afa43442ab99e961a052ca969d 2024-11-14T09:16:50,384 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/ee8b9b11c5814d3bbc1a3d54b4ac8a2b to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/ee8b9b11c5814d3bbc1a3d54b4ac8a2b 2024-11-14T09:16:50,385 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/98b4639f19fd48a195cef8d62878dcf7 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/98b4639f19fd48a195cef8d62878dcf7 2024-11-14T09:16:50,385 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/3313320d349d453198100d89ec9ce8c1 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/3313320d349d453198100d89ec9ce8c1 2024-11-14T09:16:50,386 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/5c55161fd10e4bd5a82bacc8faba0bb4 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/5c55161fd10e4bd5a82bacc8faba0bb4 2024-11-14T09:16:50,387 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/54b3756177fd4d8281dc31d657ef607d to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/54b3756177fd4d8281dc31d657ef607d 2024-11-14T09:16:50,387 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/1cb2c63c4888404d8e0b5ff435f300c0 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/1cb2c63c4888404d8e0b5ff435f300c0 2024-11-14T09:16:50,388 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/d54e098e94ef4e45a1f1eb3e9f5dbe3c to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/d54e098e94ef4e45a1f1eb3e9f5dbe3c 2024-11-14T09:16:50,389 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/0e2fd4c37d104ae693123f54b2437055 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/0e2fd4c37d104ae693123f54b2437055 2024-11-14T09:16:50,390 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/f7e065fac6364702a452ef75d6dc0121 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/f7e065fac6364702a452ef75d6dc0121 2024-11-14T09:16:50,390 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/ff79faf6140a421cb4796ac25a9faacc to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/ff79faf6140a421cb4796ac25a9faacc 2024-11-14T09:16:50,391 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/7761a02d17c64f6d874518fb4c179568 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/7761a02d17c64f6d874518fb4c179568 2024-11-14T09:16:50,392 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/420b9f987921497e979e00b0e68eb8e4 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/420b9f987921497e979e00b0e68eb8e4 2024-11-14T09:16:50,393 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/29328a346cf949d2aa4d90a1cd50506c to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/29328a346cf949d2aa4d90a1cd50506c 2024-11-14T09:16:50,394 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/4467ee897e7049ff9cb6ae12f363e9da to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/4467ee897e7049ff9cb6ae12f363e9da 2024-11-14T09:16:50,394 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/624e5540b0214f348a422fc18aa6afc8 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/624e5540b0214f348a422fc18aa6afc8 2024-11-14T09:16:50,395 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/6c10e8a0b3d840da90aca0d2b977f664 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/6c10e8a0b3d840da90aca0d2b977f664 2024-11-14T09:16:50,396 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/ea3412a382f347ada86d3eef557bbd1d to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/ea3412a382f347ada86d3eef557bbd1d 2024-11-14T09:16:50,397 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/e5e0791964a04c26bb6cb041048cc4de to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/e5e0791964a04c26bb6cb041048cc4de 2024-11-14T09:16:50,397 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/e6e35d5bdd02408082798a796bdf7b42 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/e6e35d5bdd02408082798a796bdf7b42 2024-11-14T09:16:50,398 DEBUG [StoreCloser-TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/e0614f3e9d1349be8d1a7b14483c49b0 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/e0614f3e9d1349be8d1a7b14483c49b0 2024-11-14T09:16:50,402 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/recovered.edits/368.seqid, newMaxSeqId=368, maxSeqId=4 2024-11-14T09:16:50,402 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. 2024-11-14T09:16:50,402 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1676): Region close journal for 1d18b03686124f021810b831ea32e069: Waiting for close lock at 1731575807867Running coprocessor pre-close hooks at 1731575807867Disabling compacts and flushes for region at 1731575807867Disabling writes for close at 1731575808692 (+825 ms)Obtaining lock to block concurrent updates at 1731575808693 (+1 ms)Preparing flush snapshotting stores in 1d18b03686124f021810b831ea32e069 at 1731575808693Finished memstore snapshotting TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1731575808693Flushing stores of TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069. at 1731575808693Flushing 1d18b03686124f021810b831ea32e069/A: creating writer at 1731575808693Flushing mob file 1d18b03686124f021810b831ea32e069/A: appending metadata at 1731575808697 (+4 ms)Flushing mob file 1d18b03686124f021810b831ea32e069/A: closing flushed file at 1731575808697Flushing 1d18b03686124f021810b831ea32e069/A: appending metadata at 1731575809104 (+407 ms)Flushing 1d18b03686124f021810b831ea32e069/A: closing flushed file at 1731575809104Flushing 1d18b03686124f021810b831ea32e069/B: creating writer at 1731575809511 (+407 ms)Flushing 1d18b03686124f021810b831ea32e069/B: appending metadata at 1731575809513 (+2 ms)Flushing 1d18b03686124f021810b831ea32e069/B: closing flushed file at 1731575809513Flushing 1d18b03686124f021810b831ea32e069/C: creating writer at 1731575809920 (+407 ms)Flushing 1d18b03686124f021810b831ea32e069/C: appending metadata at 1731575809922 (+2 ms)Flushing 1d18b03686124f021810b831ea32e069/C: closing flushed file at 1731575809922Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@40f476b9: reopening flushed file at 1731575810328 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1ca34e07: reopening flushed file at 1731575810332 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4286fbb5: reopening flushed file at 1731575810334 (+2 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 1d18b03686124f021810b831ea32e069 in 1646ms, sequenceid=365, compaction requested=true at 1731575810338 (+4 ms)Writing region close event to WAL at 1731575810399 (+61 ms)Running coprocessor post-close hooks at 1731575810402 (+3 ms)Closed at 1731575810402 2024-11-14T09:16:50,404 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] handler.UnassignRegionHandler(157): Closed 1d18b03686124f021810b831ea32e069 2024-11-14T09:16:50,404 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=162 updating hbase:meta row=1d18b03686124f021810b831ea32e069, regionState=CLOSED 2024-11-14T09:16:50,405 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=163, ppid=162, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 because future has completed 2024-11-14T09:16:50,407 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-14T09:16:50,407 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; CloseRegionProcedure 1d18b03686124f021810b831ea32e069, server=83f56b55f2af,39927,1731575622393 in 2.6910 sec 2024-11-14T09:16:50,409 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=162, resume processing ppid=161 2024-11-14T09:16:50,409 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, ppid=161, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d18b03686124f021810b831ea32e069, UNASSIGN in 2.6940 sec 2024-11-14T09:16:50,411 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-14T09:16:50,411 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.6970 sec 2024-11-14T09:16:50,412 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731575810411"}]},"ts":"1731575810411"} 2024-11-14T09:16:50,413 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T09:16:50,413 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T09:16:50,414 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.7050 sec 2024-11-14T09:16:51,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-14T09:16:51,866 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:16:51,866 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-14T09:16:51,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:16:51,868 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=164, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:16:51,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-14T09:16:51,868 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=164, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:16:51,871 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069 2024-11-14T09:16:51,872 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A, FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B, FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C, FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/recovered.edits] 2024-11-14T09:16:51,874 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/3a18ef6444b84f6c8ce028943927e5d7 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/3a18ef6444b84f6c8ce028943927e5d7 2024-11-14T09:16:51,875 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/d54662451e0f4fb190358d048582f9ca to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/d54662451e0f4fb190358d048582f9ca 2024-11-14T09:16:51,875 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/f71899fd38eb43b69fe008bce9a3b7af to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/A/f71899fd38eb43b69fe008bce9a3b7af 2024-11-14T09:16:51,877 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/9b51808d2eec4b36af6d916f5f0763a9 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/9b51808d2eec4b36af6d916f5f0763a9 2024-11-14T09:16:51,878 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/ecf9431e33b54e63841b623e96ae7e3e to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/ecf9431e33b54e63841b623e96ae7e3e 2024-11-14T09:16:51,879 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/eef6d62dc3724fa3928434d03851c5ec to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/B/eef6d62dc3724fa3928434d03851c5ec 2024-11-14T09:16:51,880 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/2399542321c849f5936c7fd11ab38b83 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/2399542321c849f5936c7fd11ab38b83 2024-11-14T09:16:51,881 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/80fea942f1a0451cb668a0c980e5e272 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/80fea942f1a0451cb668a0c980e5e272 2024-11-14T09:16:51,882 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/924d4d059921457488e4b754fc266eab to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/C/924d4d059921457488e4b754fc266eab 2024-11-14T09:16:51,883 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/recovered.edits/368.seqid to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069/recovered.edits/368.seqid 2024-11-14T09:16:51,884 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/default/TestAcidGuarantees/1d18b03686124f021810b831ea32e069 2024-11-14T09:16:51,884 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T09:16:51,884 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T09:16:51,885 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-14T09:16:51,886 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411140d0a30f8389843838ad922a791f2e8b8_1d18b03686124f021810b831ea32e069 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411140d0a30f8389843838ad922a791f2e8b8_1d18b03686124f021810b831ea32e069 2024-11-14T09:16:51,887 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114281785b254da421abdaf2905e5855ed3_1d18b03686124f021810b831ea32e069 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114281785b254da421abdaf2905e5855ed3_1d18b03686124f021810b831ea32e069 2024-11-14T09:16:51,888 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411142aaef23e5c30431f872232d38cd592d5_1d18b03686124f021810b831ea32e069 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411142aaef23e5c30431f872232d38cd592d5_1d18b03686124f021810b831ea32e069 2024-11-14T09:16:51,889 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111444cdbab773f446d3be787a9b068ae885_1d18b03686124f021810b831ea32e069 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111444cdbab773f446d3be787a9b068ae885_1d18b03686124f021810b831ea32e069 2024-11-14T09:16:51,889 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114595bb88231754f38874e0b0e5d0459da_1d18b03686124f021810b831ea32e069 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114595bb88231754f38874e0b0e5d0459da_1d18b03686124f021810b831ea32e069 2024-11-14T09:16:51,890 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145a96e1c2d40044ddb85b2b480659cbb6_1d18b03686124f021810b831ea32e069 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145a96e1c2d40044ddb85b2b480659cbb6_1d18b03686124f021810b831ea32e069 2024-11-14T09:16:51,891 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411146947e054673d4ba8a964accc4e06edc6_1d18b03686124f021810b831ea32e069 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411146947e054673d4ba8a964accc4e06edc6_1d18b03686124f021810b831ea32e069 2024-11-14T09:16:51,891 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111478c04de1794247b196464d6326b6a584_1d18b03686124f021810b831ea32e069 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111478c04de1794247b196464d6326b6a584_1d18b03686124f021810b831ea32e069 2024-11-14T09:16:51,892 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411147bd4882f649147e6bd0b8a60fbcd9ed1_1d18b03686124f021810b831ea32e069 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411147bd4882f649147e6bd0b8a60fbcd9ed1_1d18b03686124f021810b831ea32e069 2024-11-14T09:16:51,893 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111481352f23c9c140b894a6cf05a53e7326_1d18b03686124f021810b831ea32e069 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111481352f23c9c140b894a6cf05a53e7326_1d18b03686124f021810b831ea32e069 2024-11-14T09:16:51,893 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114835d2a83978145a18f21794f517f8d86_1d18b03686124f021810b831ea32e069 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114835d2a83978145a18f21794f517f8d86_1d18b03686124f021810b831ea32e069 2024-11-14T09:16:51,894 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114857a64ad006b4b23984e336f0269b4f0_1d18b03686124f021810b831ea32e069 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114857a64ad006b4b23984e336f0269b4f0_1d18b03686124f021810b831ea32e069 2024-11-14T09:16:51,894 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111485d174878d1741ccb57536ffe78d036a_1d18b03686124f021810b831ea32e069 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111485d174878d1741ccb57536ffe78d036a_1d18b03686124f021810b831ea32e069 2024-11-14T09:16:51,895 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411148fd42af643ab400ebba6b6db010bf9d2_1d18b03686124f021810b831ea32e069 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411148fd42af643ab400ebba6b6db010bf9d2_1d18b03686124f021810b831ea32e069 2024-11-14T09:16:51,896 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114a681086f5f6b4bebb684fcca94d648cf_1d18b03686124f021810b831ea32e069 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114a681086f5f6b4bebb684fcca94d648cf_1d18b03686124f021810b831ea32e069 2024-11-14T09:16:51,896 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ae2c7d0053334823b30bc32bbb2607aa_1d18b03686124f021810b831ea32e069 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ae2c7d0053334823b30bc32bbb2607aa_1d18b03686124f021810b831ea32e069 2024-11-14T09:16:51,897 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c1c4ce983c3c41fd942810b77528f1f6_1d18b03686124f021810b831ea32e069 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c1c4ce983c3c41fd942810b77528f1f6_1d18b03686124f021810b831ea32e069 2024-11-14T09:16:51,898 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114dbade5bc05ae4d39b2cec94a7390d244_1d18b03686124f021810b831ea32e069 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114dbade5bc05ae4d39b2cec94a7390d244_1d18b03686124f021810b831ea32e069 2024-11-14T09:16:51,898 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114dfa3bee8126b4c59bdd4fd4d73fccdbd_1d18b03686124f021810b831ea32e069 to hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114dfa3bee8126b4c59bdd4fd4d73fccdbd_1d18b03686124f021810b831ea32e069 2024-11-14T09:16:51,898 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T09:16:51,900 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=164, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:16:51,902 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T09:16:51,903 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T09:16:51,904 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=164, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:16:51,904 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T09:16:51,904 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731575811904"}]},"ts":"9223372036854775807"} 2024-11-14T09:16:51,906 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T09:16:51,906 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 1d18b03686124f021810b831ea32e069, NAME => 'TestAcidGuarantees,,1731575781563.1d18b03686124f021810b831ea32e069.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T09:16:51,906 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T09:16:51,906 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731575811906"}]},"ts":"9223372036854775807"} 2024-11-14T09:16:51,907 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T09:16:51,908 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=164, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:16:51,909 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 42 msec 2024-11-14T09:16:51,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-14T09:16:51,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T09:16:51,975 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:16:51,985 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=251 (was 250) - Thread LEAK? -, OpenFileDescriptor=455 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=398 (was 415), ProcessCount=11 (was 11), AvailableMemoryMB=6079 (was 6181) 2024-11-14T09:16:51,985 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-14T09:16:51,985 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T09:16:51,985 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T09:16:51,985 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:16:51,985 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:16:51,985 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:16:51,985 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-14T09:16:51,985 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1219616701, stopped=false 2024-11-14T09:16:51,986 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=83f56b55f2af,45325,1731575621660 2024-11-14T09:16:51,987 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45325-0x10115c10f700000, quorum=127.0.0.1:65171, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T09:16:51,987 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39927-0x10115c10f700001, quorum=127.0.0.1:65171, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T09:16:51,987 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39927-0x10115c10f700001, quorum=127.0.0.1:65171, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:16:51,987 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45325-0x10115c10f700000, quorum=127.0.0.1:65171, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:16:51,987 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T09:16:51,988 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T09:16:51,988 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T09:16:51,988 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45325-0x10115c10f700000, quorum=127.0.0.1:65171, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T09:16:51,988 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:16:51,989 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39927-0x10115c10f700001, quorum=127.0.0.1:65171, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T09:16:51,989 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '83f56b55f2af,39927,1731575622393' ***** 2024-11-14T09:16:51,989 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-14T09:16:51,989 INFO [RS:0;83f56b55f2af:39927 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-14T09:16:51,990 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-14T09:16:51,990 INFO [RS:0;83f56b55f2af:39927 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-14T09:16:51,990 INFO [RS:0;83f56b55f2af:39927 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-14T09:16:51,990 INFO [RS:0;83f56b55f2af:39927 {}] regionserver.HRegionServer(959): stopping server 83f56b55f2af,39927,1731575622393 2024-11-14T09:16:51,990 INFO [RS:0;83f56b55f2af:39927 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T09:16:51,990 INFO [RS:0;83f56b55f2af:39927 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;83f56b55f2af:39927. 2024-11-14T09:16:51,990 DEBUG [RS:0;83f56b55f2af:39927 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T09:16:51,990 DEBUG [RS:0;83f56b55f2af:39927 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:16:51,991 INFO [RS:0;83f56b55f2af:39927 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-14T09:16:51,991 INFO [RS:0;83f56b55f2af:39927 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-14T09:16:51,991 INFO [RS:0;83f56b55f2af:39927 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-14T09:16:51,991 INFO [RS:0;83f56b55f2af:39927 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-14T09:16:51,991 INFO [RS:0;83f56b55f2af:39927 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-14T09:16:51,991 DEBUG [RS:0;83f56b55f2af:39927 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-14T09:16:51,992 DEBUG [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T09:16:51,992 INFO [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T09:16:51,992 DEBUG [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T09:16:51,992 DEBUG [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T09:16:51,992 DEBUG [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T09:16:51,992 DEBUG [RS:0;83f56b55f2af:39927 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T09:16:51,992 INFO [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-14T09:16:52,009 INFO [regionserver/83f56b55f2af:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T09:16:52,016 DEBUG [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/hbase/meta/1588230740/.tmp/info/2d2749b33cf64f5f9969b1834fc423f2 is 95, key is TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83./info:/1731575654136/DeleteFamily/seqid=0 2024-11-14T09:16:52,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742448_1624 (size=6514) 2024-11-14T09:16:52,114 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-14T09:16:52,192 DEBUG [RS:0;83f56b55f2af:39927 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T09:16:52,392 DEBUG [RS:0;83f56b55f2af:39927 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T09:16:52,419 INFO [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/hbase/meta/1588230740/.tmp/info/2d2749b33cf64f5f9969b1834fc423f2 2024-11-14T09:16:52,441 DEBUG [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/hbase/meta/1588230740/.tmp/ns/5a30b2c4a3bd43d583cc132d7d558dc8 is 93, key is TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83./ns:/1731575654095/DeleteFamily/seqid=0 2024-11-14T09:16:52,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742449_1625 (size=5906) 2024-11-14T09:16:52,505 INFO [regionserver/83f56b55f2af:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-14T09:16:52,505 INFO [regionserver/83f56b55f2af:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-14T09:16:52,592 DEBUG [RS:0;83f56b55f2af:39927 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T09:16:52,793 DEBUG [RS:0;83f56b55f2af:39927 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T09:16:52,845 INFO [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/hbase/meta/1588230740/.tmp/ns/5a30b2c4a3bd43d583cc132d7d558dc8 2024-11-14T09:16:52,863 DEBUG [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/hbase/meta/1588230740/.tmp/rep_barrier/516911cf9d70458aa6f8d76d799e9801 is 102, key is TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83./rep_barrier:/1731575654095/DeleteFamily/seqid=0 2024-11-14T09:16:52,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742450_1626 (size=6025) 2024-11-14T09:16:52,993 DEBUG [RS:0;83f56b55f2af:39927 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T09:16:53,193 DEBUG [RS:0;83f56b55f2af:39927 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T09:16:53,267 INFO [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/hbase/meta/1588230740/.tmp/rep_barrier/516911cf9d70458aa6f8d76d799e9801 2024-11-14T09:16:53,284 DEBUG [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/hbase/meta/1588230740/.tmp/table/cbde714a45b442279054b4c025afe776 is 96, key is TestAcidGuarantees,,1731575625640.a92123abacf02fce4f65c3ba94256e83./table:/1731575654095/DeleteFamily/seqid=0 2024-11-14T09:16:53,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742451_1627 (size=5936) 2024-11-14T09:16:53,393 DEBUG [RS:0;83f56b55f2af:39927 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T09:16:53,593 DEBUG [RS:0;83f56b55f2af:39927 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T09:16:53,687 INFO [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/hbase/meta/1588230740/.tmp/table/cbde714a45b442279054b4c025afe776 2024-11-14T09:16:53,691 DEBUG [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/hbase/meta/1588230740/.tmp/info/2d2749b33cf64f5f9969b1834fc423f2 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/hbase/meta/1588230740/info/2d2749b33cf64f5f9969b1834fc423f2 2024-11-14T09:16:53,693 INFO [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/hbase/meta/1588230740/info/2d2749b33cf64f5f9969b1834fc423f2, entries=12, sequenceid=90, filesize=6.4 K 2024-11-14T09:16:53,694 DEBUG [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/hbase/meta/1588230740/.tmp/ns/5a30b2c4a3bd43d583cc132d7d558dc8 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/hbase/meta/1588230740/ns/5a30b2c4a3bd43d583cc132d7d558dc8 2024-11-14T09:16:53,696 INFO [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/hbase/meta/1588230740/ns/5a30b2c4a3bd43d583cc132d7d558dc8, entries=8, sequenceid=90, filesize=5.8 K 2024-11-14T09:16:53,697 DEBUG [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/hbase/meta/1588230740/.tmp/rep_barrier/516911cf9d70458aa6f8d76d799e9801 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/hbase/meta/1588230740/rep_barrier/516911cf9d70458aa6f8d76d799e9801 2024-11-14T09:16:53,699 INFO [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/hbase/meta/1588230740/rep_barrier/516911cf9d70458aa6f8d76d799e9801, entries=6, sequenceid=90, filesize=5.9 K 2024-11-14T09:16:53,699 DEBUG [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/hbase/meta/1588230740/.tmp/table/cbde714a45b442279054b4c025afe776 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/hbase/meta/1588230740/table/cbde714a45b442279054b4c025afe776 2024-11-14T09:16:53,702 INFO [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/hbase/meta/1588230740/table/cbde714a45b442279054b4c025afe776, entries=7, sequenceid=90, filesize=5.8 K 2024-11-14T09:16:53,702 INFO [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1710ms, sequenceid=90, compaction requested=false 2024-11-14T09:16:53,706 DEBUG [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-14T09:16:53,707 DEBUG [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T09:16:53,707 INFO [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T09:16:53,707 DEBUG [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731575811991Running coprocessor pre-close hooks at 1731575811992 (+1 ms)Disabling compacts and flushes for region at 1731575811992Disabling writes for close at 1731575811992Obtaining lock to block concurrent updates at 1731575811992Preparing flush snapshotting stores in 1588230740 at 1731575811992Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731575811992Flushing stores of hbase:meta,,1.1588230740 at 1731575811993 (+1 ms)Flushing 1588230740/info: creating writer at 1731575811993Flushing 1588230740/info: appending metadata at 1731575812015 (+22 ms)Flushing 1588230740/info: closing flushed file at 1731575812015Flushing 1588230740/ns: creating writer at 1731575812423 (+408 ms)Flushing 1588230740/ns: appending metadata at 1731575812441 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1731575812441Flushing 1588230740/rep_barrier: creating writer at 1731575812848 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731575812863 (+15 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731575812863Flushing 1588230740/table: creating writer at 1731575813270 (+407 ms)Flushing 1588230740/table: appending metadata at 1731575813284 (+14 ms)Flushing 1588230740/table: closing flushed file at 1731575813284Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e5d56e0: reopening flushed file at 1731575813690 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b2933ec: reopening flushed file at 1731575813693 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@32f35b7a: reopening flushed file at 1731575813696 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4bd33659: reopening flushed file at 1731575813699 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1710ms, sequenceid=90, compaction requested=false at 1731575813703 (+4 ms)Writing region close event to WAL at 1731575813704 (+1 ms)Running coprocessor post-close hooks at 1731575813707 (+3 ms)Closed at 1731575813707 2024-11-14T09:16:53,707 DEBUG [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-14T09:16:53,794 INFO [RS:0;83f56b55f2af:39927 {}] regionserver.HRegionServer(976): stopping server 83f56b55f2af,39927,1731575622393; all regions closed. 2024-11-14T09:16:53,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741834_1010 (size=25344) 2024-11-14T09:16:53,800 DEBUG [RS:0;83f56b55f2af:39927 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/oldWALs 2024-11-14T09:16:53,800 INFO [RS:0;83f56b55f2af:39927 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 83f56b55f2af%2C39927%2C1731575622393.meta:.meta(num 1731575625165) 2024-11-14T09:16:53,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741833_1009 (size=17871041) 2024-11-14T09:16:53,803 DEBUG [RS:0;83f56b55f2af:39927 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/oldWALs 2024-11-14T09:16:53,803 INFO [RS:0;83f56b55f2af:39927 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 83f56b55f2af%2C39927%2C1731575622393:(num 1731575624653) 2024-11-14T09:16:53,803 DEBUG [RS:0;83f56b55f2af:39927 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:16:53,803 INFO [RS:0;83f56b55f2af:39927 {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T09:16:53,803 INFO [RS:0;83f56b55f2af:39927 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T09:16:53,803 INFO [RS:0;83f56b55f2af:39927 {}] hbase.ChoreService(370): Chore service for: regionserver/83f56b55f2af:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-14T09:16:53,804 INFO [RS:0;83f56b55f2af:39927 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T09:16:53,804 INFO [regionserver/83f56b55f2af:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T09:16:53,804 INFO [RS:0;83f56b55f2af:39927 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39927 2024-11-14T09:16:53,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39927-0x10115c10f700001, quorum=127.0.0.1:65171, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/83f56b55f2af,39927,1731575622393 2024-11-14T09:16:53,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45325-0x10115c10f700000, quorum=127.0.0.1:65171, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T09:16:53,808 INFO [RS:0;83f56b55f2af:39927 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T09:16:53,811 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [83f56b55f2af,39927,1731575622393] 2024-11-14T09:16:53,813 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/83f56b55f2af,39927,1731575622393 already deleted, retry=false 2024-11-14T09:16:53,814 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 83f56b55f2af,39927,1731575622393 expired; onlineServers=0 2024-11-14T09:16:53,814 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '83f56b55f2af,45325,1731575621660' ***** 2024-11-14T09:16:53,814 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-14T09:16:53,814 INFO [M:0;83f56b55f2af:45325 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T09:16:53,814 INFO [M:0;83f56b55f2af:45325 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T09:16:53,814 DEBUG [M:0;83f56b55f2af:45325 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-14T09:16:53,814 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-14T09:16:53,814 DEBUG [M:0;83f56b55f2af:45325 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-14T09:16:53,814 DEBUG [master/83f56b55f2af:0:becomeActiveMaster-HFileCleaner.small.0-1731575624315 {}] cleaner.HFileCleaner(306): Exit Thread[master/83f56b55f2af:0:becomeActiveMaster-HFileCleaner.small.0-1731575624315,5,FailOnTimeoutGroup] 2024-11-14T09:16:53,814 DEBUG [master/83f56b55f2af:0:becomeActiveMaster-HFileCleaner.large.0-1731575624314 {}] cleaner.HFileCleaner(306): Exit Thread[master/83f56b55f2af:0:becomeActiveMaster-HFileCleaner.large.0-1731575624314,5,FailOnTimeoutGroup] 2024-11-14T09:16:53,814 INFO [M:0;83f56b55f2af:45325 {}] hbase.ChoreService(370): Chore service for: master/83f56b55f2af:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-14T09:16:53,814 INFO [M:0;83f56b55f2af:45325 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T09:16:53,815 DEBUG [M:0;83f56b55f2af:45325 {}] master.HMaster(1795): Stopping service threads 2024-11-14T09:16:53,815 INFO [M:0;83f56b55f2af:45325 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-14T09:16:53,815 INFO [M:0;83f56b55f2af:45325 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T09:16:53,815 ERROR [M:0;83f56b55f2af:45325 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1102635514) connection to localhost/127.0.0.1:44851 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:44851,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-14T09:16:53,815 INFO [M:0;83f56b55f2af:45325 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T09:16:53,815 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-14T09:16:53,816 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45325-0x10115c10f700000, quorum=127.0.0.1:65171, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-14T09:16:53,816 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45325-0x10115c10f700000, quorum=127.0.0.1:65171, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:16:53,816 DEBUG [M:0;83f56b55f2af:45325 {}] zookeeper.ZKUtil(347): master:45325-0x10115c10f700000, quorum=127.0.0.1:65171, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-14T09:16:53,816 WARN [M:0;83f56b55f2af:45325 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-14T09:16:53,817 INFO [M:0;83f56b55f2af:45325 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/.lastflushedseqids 2024-11-14T09:16:53,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742452_1628 (size=72) 2024-11-14T09:16:53,911 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39927-0x10115c10f700001, quorum=127.0.0.1:65171, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T09:16:53,911 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39927-0x10115c10f700001, quorum=127.0.0.1:65171, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T09:16:53,912 INFO [RS:0;83f56b55f2af:39927 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T09:16:53,912 INFO [RS:0;83f56b55f2af:39927 {}] regionserver.HRegionServer(1031): Exiting; stopping=83f56b55f2af,39927,1731575622393; zookeeper connection closed. 2024-11-14T09:16:53,912 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@27003d2c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@27003d2c 2024-11-14T09:16:53,913 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-14T09:16:54,226 INFO [M:0;83f56b55f2af:45325 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T09:16:54,227 INFO [M:0;83f56b55f2af:45325 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-14T09:16:54,227 DEBUG [M:0;83f56b55f2af:45325 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T09:16:54,227 INFO [M:0;83f56b55f2af:45325 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T09:16:54,227 DEBUG [M:0;83f56b55f2af:45325 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T09:16:54,227 DEBUG [M:0;83f56b55f2af:45325 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T09:16:54,227 DEBUG [M:0;83f56b55f2af:45325 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T09:16:54,227 INFO [M:0;83f56b55f2af:45325 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=768.12 KB heapSize=940.67 KB 2024-11-14T09:16:54,242 DEBUG [M:0;83f56b55f2af:45325 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3407596e50da4eae8c829ef1b639dc19 is 82, key is hbase:meta,,1/info:regioninfo/1731575625255/Put/seqid=0 2024-11-14T09:16:54,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742453_1629 (size=5672) 2024-11-14T09:16:54,646 INFO [M:0;83f56b55f2af:45325 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2133 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3407596e50da4eae8c829ef1b639dc19 2024-11-14T09:16:54,666 DEBUG [M:0;83f56b55f2af:45325 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/45fe5d98d9304986b34e4d5ac05aa686 is 2481, key is \x00\x00\x00\x00\x00\x00\x00\x8B/proc:d/1731575784632/Put/seqid=0 2024-11-14T09:16:54,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742454_1630 (size=44375) 2024-11-14T09:16:54,669 INFO [M:0;83f56b55f2af:45325 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=767.57 KB at sequenceid=2133 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/45fe5d98d9304986b34e4d5ac05aa686 2024-11-14T09:16:54,673 INFO [M:0;83f56b55f2af:45325 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 45fe5d98d9304986b34e4d5ac05aa686 2024-11-14T09:16:54,692 DEBUG [M:0;83f56b55f2af:45325 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5340c76c967d4acc88ee57d37aa21cbd is 69, key is 83f56b55f2af,39927,1731575622393/rs:state/1731575624425/Put/seqid=0 2024-11-14T09:16:54,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073742455_1631 (size=5156) 2024-11-14T09:16:55,096 INFO [M:0;83f56b55f2af:45325 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2133 (bloomFilter=true), to=hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5340c76c967d4acc88ee57d37aa21cbd 2024-11-14T09:16:55,099 DEBUG [M:0;83f56b55f2af:45325 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3407596e50da4eae8c829ef1b639dc19 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/3407596e50da4eae8c829ef1b639dc19 2024-11-14T09:16:55,102 INFO [M:0;83f56b55f2af:45325 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/3407596e50da4eae8c829ef1b639dc19, entries=8, sequenceid=2133, filesize=5.5 K 2024-11-14T09:16:55,102 DEBUG [M:0;83f56b55f2af:45325 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/45fe5d98d9304986b34e4d5ac05aa686 as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/45fe5d98d9304986b34e4d5ac05aa686 2024-11-14T09:16:55,105 INFO [M:0;83f56b55f2af:45325 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 45fe5d98d9304986b34e4d5ac05aa686 2024-11-14T09:16:55,105 INFO [M:0;83f56b55f2af:45325 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/45fe5d98d9304986b34e4d5ac05aa686, entries=164, sequenceid=2133, filesize=43.3 K 2024-11-14T09:16:55,105 DEBUG [M:0;83f56b55f2af:45325 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5340c76c967d4acc88ee57d37aa21cbd as hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5340c76c967d4acc88ee57d37aa21cbd 2024-11-14T09:16:55,107 INFO [M:0;83f56b55f2af:45325 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44851/user/jenkins/test-data/9d2e26f5-f2cc-da78-c334-bec7dd3284db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5340c76c967d4acc88ee57d37aa21cbd, entries=1, sequenceid=2133, filesize=5.0 K 2024-11-14T09:16:55,108 INFO [M:0;83f56b55f2af:45325 {}] regionserver.HRegion(3140): Finished flush of dataSize ~768.12 KB/786556, heapSize ~940.38 KB/962944, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 881ms, sequenceid=2133, compaction requested=false 2024-11-14T09:16:55,109 INFO [M:0;83f56b55f2af:45325 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T09:16:55,109 DEBUG [M:0;83f56b55f2af:45325 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731575814227Disabling compacts and flushes for region at 1731575814227Disabling writes for close at 1731575814227Obtaining lock to block concurrent updates at 1731575814227Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731575814227Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=786556, getHeapSize=963184, getOffHeapSize=0, getCellsCount=2322 at 1731575814227Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731575814228 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731575814228Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731575814242 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731575814242Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731575814649 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731575814665 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731575814665Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731575814673 (+8 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731575814692 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731575814692Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@62cb5c4d: reopening flushed file at 1731575815099 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26468b14: reopening flushed file at 1731575815102 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e6354df: reopening flushed file at 1731575815105 (+3 ms)Finished flush of dataSize ~768.12 KB/786556, heapSize ~940.38 KB/962944, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 881ms, sequenceid=2133, compaction requested=false at 1731575815108 (+3 ms)Writing region close event to WAL at 1731575815109 (+1 ms)Closed at 1731575815109 2024-11-14T09:16:55,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741830_1006 (size=926376) 2024-11-14T09:16:55,113 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T09:16:55,113 INFO [M:0;83f56b55f2af:45325 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-14T09:16:55,113 INFO [M:0;83f56b55f2af:45325 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45325 2024-11-14T09:16:55,113 INFO [M:0;83f56b55f2af:45325 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T09:16:55,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45325-0x10115c10f700000, quorum=127.0.0.1:65171, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T09:16:55,215 INFO [M:0;83f56b55f2af:45325 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T09:16:55,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45325-0x10115c10f700000, quorum=127.0.0.1:65171, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T09:16:55,220 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T09:16:55,223 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T09:16:55,223 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T09:16:55,223 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T09:16:55,223 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae915b1d-3a0c-f123-b058-6fcd2c888fa8/hadoop.log.dir/,STOPPED} 2024-11-14T09:16:55,226 WARN [BP-784109664-172.17.0.2-1731575615976 heartbeating to localhost/127.0.0.1:44851 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-14T09:16:55,226 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-14T09:16:55,226 WARN [BP-784109664-172.17.0.2-1731575615976 heartbeating to localhost/127.0.0.1:44851 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-784109664-172.17.0.2-1731575615976 (Datanode Uuid 1fc8a527-2c66-4742-bab0-e6bcd0d07540) service to localhost/127.0.0.1:44851 2024-11-14T09:16:55,226 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-14T09:16:55,228 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae915b1d-3a0c-f123-b058-6fcd2c888fa8/cluster_e7e5a44a-d236-23e3-0b04-c3f634c73d33/data/data1/current/BP-784109664-172.17.0.2-1731575615976 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T09:16:55,228 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae915b1d-3a0c-f123-b058-6fcd2c888fa8/cluster_e7e5a44a-d236-23e3-0b04-c3f634c73d33/data/data2/current/BP-784109664-172.17.0.2-1731575615976 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T09:16:55,229 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-14T09:16:55,236 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-14T09:16:55,236 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T09:16:55,236 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T09:16:55,237 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T09:16:55,237 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae915b1d-3a0c-f123-b058-6fcd2c888fa8/hadoop.log.dir/,STOPPED} 2024-11-14T09:16:55,252 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-14T09:16:55,367 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down